From 295be71f00fe84ad686ac7a7eef00752d268522a Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Tue, 10 Aug 2021 14:19:26 +0300 Subject: [PATCH] [#206] api: Add objects cache Signed-off-by: Angira Kekteeva --- api/cache/object_cache.go | 62 +++++++++++++++++++++++++++++++++++++++ api/layer/layer.go | 22 ++++++++++---- api/layer/object.go | 5 ++++ go.mod | 1 + go.sum | 2 ++ 5 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 api/cache/object_cache.go diff --git a/api/cache/object_cache.go b/api/cache/object_cache.go new file mode 100644 index 00000000..58a87101 --- /dev/null +++ b/api/cache/object_cache.go @@ -0,0 +1,62 @@ +package cache + +import ( + "time" + + "github.com/bluele/gcache" + "github.com/nspcc-dev/neofs-api-go/pkg/object" +) + +// ObjectsCache provides interface for lru cache for objects. +type ObjectsCache interface { + Get(address *object.Address) *object.Object + Put(address *object.Address, obj object.Object) error + Delete(address *object.Address) bool +} + +const ( + // DefaultObjectsCacheLifetime is a default lifetime of objects in cache. + DefaultObjectsCacheLifetime = time.Minute * 5 + // DefaultObjectsCacheSize is a default maximum number of objects in cache. + DefaultObjectsCacheSize = 1e6 +) + +type ( + // ObjectHeadersCache contains cache with objects and lifetime of cache entries. + ObjectHeadersCache struct { + cache gcache.Cache + lifetime time.Duration + } +) + +// New creates an object of ObjectHeadersCache. +func New(cacheSize int, lifetime time.Duration) *ObjectHeadersCache { + gc := gcache.New(cacheSize).LRU().Build() + + return &ObjectHeadersCache{cache: gc, lifetime: lifetime} +} + +// Get returns cached object. +func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object { + entry, err := o.cache.Get(address.String()) + if err != nil { + return nil + } + + result, ok := entry.(object.Object) + if !ok { + return nil + } + + return &result +} + +// Put puts an object to cache. +func (o *ObjectHeadersCache) Put(address *object.Address, obj object.Object) error { + return o.cache.SetWithExpire(address.String(), obj, o.lifetime) +} + +// Delete deletes an object from cache. +func (o *ObjectHeadersCache) Delete(address *object.Address) bool { + return o.cache.Remove(address.String()) +} diff --git a/api/layer/layer.go b/api/layer/layer.go index 0f81bc4a..cebd3536 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -15,6 +15,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" @@ -26,7 +27,7 @@ type ( pool pool.Pool log *zap.Logger listObjCache ObjectsListCache - + objCache cache.ObjectsCache } // Params stores basic API parameters. @@ -134,6 +135,7 @@ func NewLayer(log *zap.Logger, conns pool.Pool) Client { pool: conns, log: log, listObjCache: newListObjectsCache(defaultObjectsListCacheLifetime), + objCache: cache.New(cache.DefaultObjectsCacheSize, cache.DefaultObjectsCacheLifetime), } } @@ -238,6 +240,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { } if err != nil { + n.objCache.Delete(addr) return fmt.Errorf("couldn't get object, cid: %s : %w", bkt.CID, err) } @@ -275,11 +278,20 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string) addr.SetObjectID(oid) addr.SetContainerID(bkt.CID) - if meta, err = n.objectHead(ctx, addr); err != nil { - n.log.Error("could not fetch object head", zap.Error(err)) - return nil, err + /* todo: now we get an address via request to NeoFS and try to find the object with the address in cache + but it will be resolved after implementation of local cache with nicenames and address of objects + for get/head requests */ + meta = n.objCache.Get(addr) + if meta == nil { + meta, err = n.objectHead(ctx, addr) + if err != nil { + n.log.Error("could not fetch object head", zap.Error(err)) + return nil, err + } + if err = n.objCache.Put(addr, *meta); err != nil { + n.log.Error("couldn't cache an object", zap.Error(err)) + } } - return objectInfoFromMeta(bkt, meta, "", ""), nil } diff --git a/api/layer/object.go b/api/layer/object.go index 7f90f87f..77d310d1 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -183,6 +183,10 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, return nil, err } + if err = n.objCache.Put(addr, *meta); err != nil { + n.log.Error("couldn't cache an object", zap.Error(err)) + } + return &ObjectInfo{ id: oid, @@ -201,6 +205,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, func (n *layer) objectDelete(ctx context.Context, address *object.Address) error { dop := new(client.DeleteObjectParams) dop.WithAddress(address) + n.objCache.Delete(address) return n.pool.DeleteObject(ctx, dop, n.BearerOpt(ctx)) } diff --git a/go.mod b/go.mod index 856f54fb..e07ddea6 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/alecthomas/participle v0.7.1 // indirect github.com/aws/aws-sdk-go v1.37.9 + github.com/bluele/gcache v0.0.2 github.com/google/uuid v1.2.0 github.com/gorilla/mux v1.8.0 github.com/nspcc-dev/neo-go v0.95.3 diff --git a/go.sum b/go.sum index 717f3422..9c11bbd9 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=