[#206] api: Add objects cache

Signed-off-by: Angira Kekteeva <kira@nspcc.ru>
This commit is contained in:
Angira Kekteeva 2021-08-10 14:19:26 +03:00 committed by Kirillov Denis
parent ae19eb4ad4
commit 295be71f00
5 changed files with 87 additions and 5 deletions

62
api/cache/object_cache.go vendored Normal file
View file

@ -0,0 +1,62 @@
package cache
import (
"time"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
)
// ObjectsCache provides interface for lru cache for objects.
type ObjectsCache interface {
Get(address *object.Address) *object.Object
Put(address *object.Address, obj object.Object) error
Delete(address *object.Address) bool
}
const (
// DefaultObjectsCacheLifetime is a default lifetime of objects in cache.
DefaultObjectsCacheLifetime = time.Minute * 5
// DefaultObjectsCacheSize is a default maximum number of objects in cache.
DefaultObjectsCacheSize = 1e6
)
type (
// ObjectHeadersCache contains cache with objects and lifetime of cache entries.
ObjectHeadersCache struct {
cache gcache.Cache
lifetime time.Duration
}
)
// New creates an object of ObjectHeadersCache.
func New(cacheSize int, lifetime time.Duration) *ObjectHeadersCache {
gc := gcache.New(cacheSize).LRU().Build()
return &ObjectHeadersCache{cache: gc, lifetime: lifetime}
}
// Get returns cached object.
func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object {
entry, err := o.cache.Get(address.String())
if err != nil {
return nil
}
result, ok := entry.(object.Object)
if !ok {
return nil
}
return &result
}
// Put puts an object to cache.
func (o *ObjectHeadersCache) Put(address *object.Address, obj object.Object) error {
return o.cache.SetWithExpire(address.String(), obj, o.lifetime)
}
// Delete deletes an object from cache.
func (o *ObjectHeadersCache) Delete(address *object.Address) bool {
return o.cache.Remove(address.String())
}

View file

@ -15,6 +15,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool"
@ -26,7 +27,7 @@ type (
pool pool.Pool pool pool.Pool
log *zap.Logger log *zap.Logger
listObjCache ObjectsListCache listObjCache ObjectsListCache
objCache cache.ObjectsCache
} }
// Params stores basic API parameters. // Params stores basic API parameters.
@ -134,6 +135,7 @@ func NewLayer(log *zap.Logger, conns pool.Pool) Client {
pool: conns, pool: conns,
log: log, log: log,
listObjCache: newListObjectsCache(defaultObjectsListCacheLifetime), listObjCache: newListObjectsCache(defaultObjectsListCacheLifetime),
objCache: cache.New(cache.DefaultObjectsCacheSize, cache.DefaultObjectsCacheLifetime),
} }
} }
@ -238,6 +240,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
} }
if err != nil { if err != nil {
n.objCache.Delete(addr)
return fmt.Errorf("couldn't get object, cid: %s : %w", bkt.CID, err) return fmt.Errorf("couldn't get object, cid: %s : %w", bkt.CID, err)
} }
@ -275,11 +278,20 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string)
addr.SetObjectID(oid) addr.SetObjectID(oid)
addr.SetContainerID(bkt.CID) addr.SetContainerID(bkt.CID)
if meta, err = n.objectHead(ctx, addr); err != nil { /* todo: now we get an address via request to NeoFS and try to find the object with the address in cache
but it will be resolved after implementation of local cache with nicenames and address of objects
for get/head requests */
meta = n.objCache.Get(addr)
if meta == nil {
meta, err = n.objectHead(ctx, addr)
if err != nil {
n.log.Error("could not fetch object head", zap.Error(err)) n.log.Error("could not fetch object head", zap.Error(err))
return nil, err return nil, err
} }
if err = n.objCache.Put(addr, *meta); err != nil {
n.log.Error("couldn't cache an object", zap.Error(err))
}
}
return objectInfoFromMeta(bkt, meta, "", ""), nil return objectInfoFromMeta(bkt, meta, "", ""), nil
} }

View file

@ -183,6 +183,10 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
return nil, err return nil, err
} }
if err = n.objCache.Put(addr, *meta); err != nil {
n.log.Error("couldn't cache an object", zap.Error(err))
}
return &ObjectInfo{ return &ObjectInfo{
id: oid, id: oid,
@ -201,6 +205,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
func (n *layer) objectDelete(ctx context.Context, address *object.Address) error { func (n *layer) objectDelete(ctx context.Context, address *object.Address) error {
dop := new(client.DeleteObjectParams) dop := new(client.DeleteObjectParams)
dop.WithAddress(address) dop.WithAddress(address)
n.objCache.Delete(address)
return n.pool.DeleteObject(ctx, dop, n.BearerOpt(ctx)) return n.pool.DeleteObject(ctx, dop, n.BearerOpt(ctx))
} }

1
go.mod
View file

@ -5,6 +5,7 @@ go 1.16
require ( require (
github.com/alecthomas/participle v0.7.1 // indirect github.com/alecthomas/participle v0.7.1 // indirect
github.com/aws/aws-sdk-go v1.37.9 github.com/aws/aws-sdk-go v1.37.9
github.com/bluele/gcache v0.0.2
github.com/google/uuid v1.2.0 github.com/google/uuid v1.2.0
github.com/gorilla/mux v1.8.0 github.com/gorilla/mux v1.8.0
github.com/nspcc-dev/neo-go v0.95.3 github.com/nspcc-dev/neo-go v0.95.3

2
go.sum
View file

@ -64,6 +64,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw=
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=