[#122] Add versioning cache

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-08-18 16:48:58 +03:00
parent f6c51cc9ee
commit 11558124cd
15 changed files with 503 additions and 203 deletions

64
api/cache/bucket.go vendored Normal file
View file

@ -0,0 +1,64 @@
package cache
import (
"time"
"github.com/bluele/gcache"
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
)
type (
// BucketCache provides interface for lru cache for objects.
BucketCache interface {
Get(key string) *BucketInfo
Put(bkt *BucketInfo) error
Delete(key string) bool
}
// BucketInfo stores basic bucket data.
BucketInfo struct {
Name string
CID *cid.ID
Owner *owner.ID
Created time.Time
}
// GetBucketCache contains cache with objects and lifetime of cache entries.
GetBucketCache struct {
cache gcache.Cache
lifetime time.Duration
}
)
// NewBucketCache creates an object of BucketCache.
func NewBucketCache(cacheSize int, lifetime time.Duration) *GetBucketCache {
gc := gcache.New(cacheSize).LRU().Build()
return &GetBucketCache{cache: gc, lifetime: lifetime}
}
// Get returns cached object.
func (o *GetBucketCache) Get(key string) *BucketInfo {
entry, err := o.cache.Get(key)
if err != nil {
return nil
}
result, ok := entry.(*BucketInfo)
if !ok {
return nil
}
return result
}
// Put puts an object to cache.
func (o *GetBucketCache) Put(bkt *BucketInfo) error {
return o.cache.SetWithExpire(bkt.Name, bkt, o.lifetime)
}
// Delete deletes an object from cache.
func (o *GetBucketCache) Delete(key string) bool {
return o.cache.Remove(key)
}

55
api/cache/head_cache.go vendored Normal file
View file

@ -0,0 +1,55 @@
package cache
import (
"time"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
)
// HeadObjectsCache provides interface for lru cache for objects.
type HeadObjectsCache interface {
Get(key string) *object.Address
Put(key string, address *object.Address) error
Delete(key string) bool
}
type (
// HeadObjectCache contains cache with objects and lifetime of cache entries.
HeadObjectCache struct {
cache gcache.Cache
lifetime time.Duration
}
)
// NewHeadObject creates an object of ObjectHeadersCache.
func NewHeadObject(cacheSize int, lifetime time.Duration) *HeadObjectCache {
gc := gcache.New(cacheSize).LRU().Build()
return &HeadObjectCache{cache: gc, lifetime: lifetime}
}
// Get returns cached object.
func (o *HeadObjectCache) Get(key string) *object.Address {
entry, err := o.cache.Get(key)
if err != nil {
return nil
}
result, ok := entry.(*object.Address)
if !ok {
return nil
}
return result
}
// Put puts an object to cache.
func (o *HeadObjectCache) Put(key string, address *object.Address) error {
return o.cache.SetWithExpire(key, address, o.lifetime)
}
// Delete deletes an object from cache.
func (o *HeadObjectCache) Delete(key string) bool {
return o.cache.Remove(key)
}

View file

@ -10,7 +10,7 @@ import (
// ObjectsCache provides interface for lru cache for objects. // ObjectsCache provides interface for lru cache for objects.
type ObjectsCache interface { type ObjectsCache interface {
Get(address *object.Address) *object.Object Get(address *object.Address) *object.Object
Put(address *object.Address, obj object.Object) error Put(obj object.Object) error
Delete(address *object.Address) bool Delete(address *object.Address) bool
} }
@ -52,8 +52,8 @@ func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object {
} }
// Put puts an object to cache. // Put puts an object to cache.
func (o *ObjectHeadersCache) Put(address *object.Address, obj object.Object) error { func (o *ObjectHeadersCache) Put(obj object.Object) error {
return o.cache.SetWithExpire(address.String(), obj, o.lifetime) return o.cache.SetWithExpire(obj.ContainerID().String()+"/"+obj.ID().String(), obj, o.lifetime)
} }
// Delete deletes an object from cache. // Delete deletes an object from cache.

View file

@ -4,6 +4,8 @@ import (
"testing" "testing"
"time" "time"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test" objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -14,23 +16,23 @@ const (
) )
func TestCache(t *testing.T) { func TestCache(t *testing.T) {
var ( obj := objecttest.Object()
address = objecttest.Address() address := object.NewAddress()
object = objecttest.Object() address.SetContainerID(obj.ContainerID())
) address.SetObjectID(obj.ID())
t.Run("check get", func(t *testing.T) { t.Run("check get", func(t *testing.T) {
cache := New(cachesize, lifetime) cache := New(cachesize, lifetime)
err := cache.Put(address, *object) err := cache.Put(*obj)
require.NoError(t, err) require.NoError(t, err)
actual := cache.Get(address) actual := cache.Get(address)
require.Equal(t, object, actual) require.Equal(t, obj, actual)
}) })
t.Run("check delete", func(t *testing.T) { t.Run("check delete", func(t *testing.T) {
cache := New(cachesize, lifetime) cache := New(cachesize, lifetime)
err := cache.Put(address, *object) err := cache.Put(*obj)
require.NoError(t, err) require.NoError(t, err)
cache.Delete(address) cache.Delete(address)

56
api/cache/system.go vendored Normal file
View file

@ -0,0 +1,56 @@
package cache
import (
"time"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/bluele/gcache"
)
type (
// SystemCache provides interface for lru cache for objects.
SystemCache interface {
Get(key string) *object.Object
Put(key string, obj *object.Object) error
Delete(key string) bool
}
// systemCache contains cache with objects and lifetime of cache entries.
systemCache struct {
cache gcache.Cache
lifetime time.Duration
}
)
// NewSystemCache creates an object of SystemCache.
func NewSystemCache(cacheSize int, lifetime time.Duration) SystemCache {
gc := gcache.New(cacheSize).LRU().Build()
return &systemCache{cache: gc, lifetime: lifetime}
}
// Get returns cached object.
func (o *systemCache) Get(key string) *object.Object {
entry, err := o.cache.Get(key)
if err != nil {
return nil
}
result, ok := entry.(*object.Object)
if !ok {
return nil
}
return result
}
// Put puts an object to cache.
func (o *systemCache) Put(key string, obj *object.Object) error {
return o.cache.SetWithExpire(key, obj, o.lifetime)
}
// Delete deletes an object from cache.
func (o *systemCache) Delete(key string) bool {
return o.cache.Remove(key)
}

View file

@ -46,18 +46,22 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "could not fetch object info", reqInfo, err) h.logAndSendError(w, "could not fetch object info", reqInfo, err)
return return
} }
buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType))
getParams := &layer.GetObjectParams{ if len(inf.ContentType) == 0 {
ObjectInfo: inf, buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType))
Writer: buffer, getParams := &layer.GetObjectParams{
Range: getRangeToDetectContentType(inf.Size), ObjectInfo: inf,
VersionID: reqInfo.URL.Query().Get("versionId"), Writer: buffer,
Range: getRangeToDetectContentType(inf.Size),
VersionID: reqInfo.URL.Query().Get("versionId"),
}
if err = h.obj.GetObject(r.Context(), getParams); err != nil {
h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", inf.ID()))
return
}
inf.ContentType = http.DetectContentType(buffer.Bytes())
} }
if err = h.obj.GetObject(r.Context(), getParams); err != nil {
h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", inf.ID()))
return
}
inf.ContentType = http.DetectContentType(buffer.Bytes())
writeHeaders(w.Header(), inf) writeHeaders(w.Header(), inf)
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
} }

View file

@ -263,7 +263,7 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck
} }
for _, prefix := range info.CommonPrefixes { for _, prefix := range info.CommonPrefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix}) res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: prefix})
} }
for _, ver := range info.Version { for _, ver := range info.Version {

View file

@ -11,37 +11,29 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
"github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/container"
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool"
"go.uber.org/zap" "go.uber.org/zap"
) )
type ( type (
// BucketInfo stores basic bucket data.
BucketInfo struct {
Name string
CID *cid.ID
Owner *owner.ID
Created time.Time
BasicACL uint32
}
// BucketACL extends BucketInfo by eacl.Table. // BucketACL extends BucketInfo by eacl.Table.
BucketACL struct { BucketACL struct {
Info *BucketInfo Info *cache.BucketInfo
EACL *eacl.Table EACL *eacl.Table
} }
) )
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*BucketInfo, error) { func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*cache.BucketInfo, error) {
var ( var (
err error err error
res *container.Container res *container.Container
rid = api.GetRequestID(ctx) rid = api.GetRequestID(ctx)
bearerOpt = n.BearerOpt(ctx) bearerOpt = n.BearerOpt(ctx)
info = &BucketInfo{ info = &cache.BucketInfo{
CID: cid, CID: cid,
Name: cid.String(), Name: cid.String(),
} }
@ -82,10 +74,17 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*BucketInfo, er
} }
} }
if err := n.bucketCache.Put(info); err != nil {
n.log.Warn("could not put bucket info into cache",
zap.Stringer("cid", cid),
zap.String("bucket_name", info.Name),
zap.Error(err))
}
return info, nil return info, nil
} }
func (n *layer) containerList(ctx context.Context) ([]*BucketInfo, error) { func (n *layer) containerList(ctx context.Context) ([]*cache.BucketInfo, error) {
var ( var (
err error err error
own = n.Owner(ctx) own = n.Owner(ctx)
@ -101,7 +100,7 @@ func (n *layer) containerList(ctx context.Context) ([]*BucketInfo, error) {
return nil, err return nil, err
} }
list := make([]*BucketInfo, 0, len(res)) list := make([]*cache.BucketInfo, 0, len(res))
for _, cid := range res { for _, cid := range res {
info, err := n.containerInfo(ctx, cid) info, err := n.containerInfo(ctx, cid)
if err != nil { if err != nil {

View file

@ -3,24 +3,56 @@ package layer
import ( import (
"io" "io"
"net/http" "net/http"
"sync"
) )
type detector struct { type (
io.Reader detector struct {
sync.Once io.Reader
err error
data []byte
}
errReader struct {
data []byte
err error
offset int
}
)
contentType string const contentTypeDetectSize = 512
func newReader(data []byte, err error) *errReader {
return &errReader{data: data, err: err}
} }
func newDetector(r io.Reader) *detector { func (r *errReader) Read(b []byte) (int, error) {
return &detector{Reader: r} if r.offset >= len(r.data) {
return 0, io.EOF
}
n := copy(b, r.data[r.offset:])
r.offset += n
if r.offset >= len(r.data) {
return n, r.err
}
return n, nil
} }
func (d *detector) Read(data []byte) (int, error) { func newDetector(reader io.Reader) *detector {
d.Do(func() { return &detector{
d.contentType = http.DetectContentType(data) data: make([]byte, contentTypeDetectSize),
}) Reader: reader,
}
return d.Reader.Read(data) }
func (d *detector) Detect() (string, error) {
n, err := d.Reader.Read(d.data)
if err != nil && err != io.EOF {
d.err = err
return "", err
}
d.data = d.data[:n]
return http.DetectContentType(d.data), nil
}
func (d *detector) MultiReader() io.Reader {
return io.MultiReader(newReader(d.data, d.err), d.Reader)
} }

View file

@ -28,10 +28,13 @@ import (
type ( type (
layer struct { layer struct {
pool pool.Pool pool pool.Pool
log *zap.Logger log *zap.Logger
listObjCache ObjectsListCache listsCache ObjectsListCache
objCache cache.ObjectsCache objCache cache.ObjectsCache
headCache cache.HeadObjectsCache
bucketCache cache.BucketCache
systemCache cache.SystemCache
} }
// CacheConfig contains params for caches. // CacheConfig contains params for caches.
@ -156,8 +159,8 @@ type (
PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error)
GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error) GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error)
ListBuckets(ctx context.Context) ([]*BucketInfo, error) ListBuckets(ctx context.Context) ([]*cache.BucketInfo, error)
GetBucketInfo(ctx context.Context, name string) (*BucketInfo, error) GetBucketInfo(ctx context.Context, name string) (*cache.BucketInfo, error)
GetBucketACL(ctx context.Context, name string) (*BucketACL, error) GetBucketACL(ctx context.Context, name string) (*BucketACL, error)
PutBucketACL(ctx context.Context, p *PutBucketACLParams) error PutBucketACL(ctx context.Context, p *PutBucketACLParams) error
CreateBucket(ctx context.Context, p *CreateBucketParams) (*cid.ID, error) CreateBucket(ctx context.Context, p *CreateBucketParams) (*cid.ID, error)
@ -228,26 +231,22 @@ func (v *objectVersions) getLast() *ObjectInfo {
return nil return nil
} }
func (v *objectVersions) getFiltered() []*ObjectVersionInfo { func (v *objectVersions) getFiltered() []*ObjectInfo {
if len(v.objects) == 0 { if len(v.objects) == 0 {
return nil return nil
} }
v.sort() v.sort()
existedVersions := getExistedVersions(v) existedVersions := getExistedVersions(v)
res := make([]*ObjectVersionInfo, 0, len(v.objects)) res := make([]*ObjectInfo, 0, len(v.objects))
for _, version := range v.objects { for _, version := range v.objects {
delMark := version.Headers[versionsDeleteMarkAttr] delMark := version.Headers[versionsDeleteMarkAttr]
if contains(existedVersions, version.Version()) && (delMark == delMarkFullObject || delMark == "") { if contains(existedVersions, version.Version()) && (delMark == delMarkFullObject || delMark == "") {
res = append(res, &ObjectVersionInfo{Object: version}) res = append(res, version)
} }
} }
if len(res) > 0 {
res[len(res)-1].IsLatest = true
}
return res return res
} }
@ -279,10 +278,14 @@ func (t *VersionedObject) String() string {
// and establishes gRPC connection with node. // and establishes gRPC connection with node.
func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client { func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
return &layer{ return &layer{
pool: conns, pool: conns,
log: log, log: log,
listObjCache: newListObjectsCache(config.ListObjectsLifetime), listsCache: newListObjectsCache(config.ListObjectsLifetime),
objCache: cache.New(config.Size, config.Lifetime), objCache: cache.New(config.Size, config.Lifetime),
//todo reconsider cache params
headCache: cache.NewHeadObject(1000, time.Minute),
bucketCache: cache.NewBucketCache(150, time.Minute),
systemCache: cache.NewSystemCache(1000, 5*time.Minute),
} }
} }
@ -320,12 +323,16 @@ func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Objec
} }
// GetBucketInfo returns bucket info by name. // GetBucketInfo returns bucket info by name.
func (n *layer) GetBucketInfo(ctx context.Context, name string) (*BucketInfo, error) { func (n *layer) GetBucketInfo(ctx context.Context, name string) (*cache.BucketInfo, error) {
name, err := url.QueryUnescape(name) name, err := url.QueryUnescape(name)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if bktInfo := n.bucketCache.Get(name); bktInfo != nil {
return bktInfo, nil
}
containerID := new(cid.ID) containerID := new(cid.ID)
if err := containerID.Parse(name); err != nil { if err := containerID.Parse(name); err != nil {
list, err := n.containerList(ctx) list, err := n.containerList(ctx)
@ -374,7 +381,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err
// ListBuckets returns all user containers. Name of the bucket is a container // ListBuckets returns all user containers. Name of the bucket is a container
// id. Timestamp is omitted since it is not saved in neofs container. // id. Timestamp is omitted since it is not saved in neofs container.
func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) { func (n *layer) ListBuckets(ctx context.Context) ([]*cache.BucketInfo, error) {
return n.containerList(ctx) return n.containerList(ctx)
} }
@ -382,21 +389,12 @@ func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) {
func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
var err error var err error
//if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
// return fmt.Errorf("couldn't find bucket: %s : %w", p.Bucket, err)
//} else if oid, err = n.objectFindID(ctx, &findParams{cid: bkt.CID, val: p.Object}); err != nil {
// return fmt.Errorf("search of the object failed: cid: %s, val: %s : %w", bkt.CID, p.Object, err)
//}
addr := object.NewAddress()
addr.SetObjectID(p.ObjectInfo.ID())
addr.SetContainerID(p.ObjectInfo.CID())
params := &getParams{ params := &getParams{
Writer: p.Writer, Writer: p.Writer,
address: addr, cid: p.ObjectInfo.CID(),
offset: p.Offset, oid: p.ObjectInfo.ID(),
length: p.Length, offset: p.Offset,
length: p.Length,
} }
if p.Range != nil { if p.Range != nil {
@ -411,7 +409,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
} }
if err != nil { if err != nil {
n.objCache.Delete(addr) n.objCache.Delete(p.ObjectInfo.Address())
return fmt.Errorf("couldn't get object, cid: %s : %w", p.ObjectInfo.CID(), err) return fmt.Errorf("couldn't get object, cid: %s : %w", p.ObjectInfo.CID(), err)
} }
@ -433,32 +431,26 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object
return n.headVersion(ctx, bkt, p.VersionID) return n.headVersion(ctx, bkt, p.VersionID)
} }
func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) { func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *cache.BucketInfo) (*ObjectInfo, error) {
if meta := n.systemCache.Get(bktVersionSettingsObject); meta != nil {
return objInfoFromMeta(bkt, meta), nil
}
oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bktVersionSettingsObject}) oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bktVersionSettingsObject})
if err != nil { if err != nil {
return nil, err return nil, err
} }
addr := object.NewAddress() meta, err := n.objectHead(ctx, bkt.CID, oid)
addr.SetObjectID(oid) if err != nil {
addr.SetContainerID(bkt.CID) n.log.Error("could not fetch object head", zap.Error(err))
return nil, err
/* todo: now we get an address via request to NeoFS and try to find the object with the address in cache }
but it will be resolved after implementation of local cache with nicenames and address of objects if err = n.systemCache.Put(bktVersionSettingsObject, meta); err != nil {
for get/head requests */ n.log.Error("couldn't cache system object", zap.Error(err))
meta := n.objCache.Get(addr)
if meta == nil {
meta, err = n.objectHead(ctx, bkt.CID, oid)
if err != nil {
n.log.Error("could not fetch object head", zap.Error(err))
return nil, err
}
if err = n.objCache.Put(addr, *meta); err != nil {
n.log.Error("couldn't cache an object", zap.Error(err))
}
} }
return objectInfoFromMeta(bkt, meta, "", ""), nil return objInfoFromMeta(bkt, meta), nil
} }
// PutObject into storage. // PutObject into storage.
@ -496,7 +488,7 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf
} }
// DeleteObject removes all objects with passed nice name. // DeleteObject removes all objects with passed nice name.
func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) error { func (n *layer) deleteObject(ctx context.Context, bkt *cache.BucketInfo, obj *VersionedObject) error {
var ( var (
err error err error
ids []*object.ID ids []*object.ID
@ -543,7 +535,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione
return nil return nil
} }
func (n *layer) checkVersionsExists(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) (*object.ID, error) { func (n *layer) checkVersionsExists(ctx context.Context, bkt *cache.BucketInfo, obj *VersionedObject) (*object.ID, error) {
id := object.NewID() id := object.NewID()
if err := id.Parse(obj.VersionID); err != nil { if err := id.Parse(obj.VersionID); err != nil {
return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()}
@ -608,60 +600,70 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
} }
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
res := ListObjectVersionsInfo{} var versions map[string]*objectVersions
versions := make(map[string]*objectVersions) res := &ListObjectVersionsInfo{}
bkt, err := n.GetBucketInfo(ctx, p.Bucket) bkt, err := n.GetBucketInfo(ctx, p.Bucket)
if err != nil { if err != nil {
return nil, err return nil, err
} }
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID})
cacheKey, err := createKey(ctx, bkt.CID, listVersionsMethod, p.Prefix, p.Delimiter)
if err != nil { if err != nil {
return nil, err return nil, err
} }
for _, id := range ids { allObjects := n.listsCache.Get(cacheKey)
meta, err := n.objectHead(ctx, bkt.CID, id) if allObjects == nil {
versions, err = n.getAllObjectsVersions(ctx, bkt, p.Prefix, p.Delimiter)
if err != nil { if err != nil {
n.log.Warn("could not fetch object meta", zap.Error(err)) return nil, err
continue
} }
if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
if oi.Name <= p.KeyMarker {
continue
}
if isSystem(oi) {
continue
}
if objVersions, ok := versions[oi.Name]; ok { sortedNames := make([]string, 0, len(versions))
objVersions.appendVersion(oi) for k := range versions {
versions[oi.Name] = objVersions sortedNames = append(sortedNames, k)
} else {
objVersion := newObjectVersions(oi.Name)
objVersion.appendVersion(oi)
versions[oi.Name] = objVersion
}
} }
sort.Strings(sortedNames)
allObjects = make([]*ObjectInfo, 0, p.MaxKeys)
for _, name := range sortedNames {
allObjects = append(allObjects, versions[name].getFiltered()...)
}
// putting to cache a copy of allObjects because allObjects can be modified further
n.listsCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...))
} }
sortedNames := make([]string, 0, len(versions)) for i, obj := range allObjects {
for k := range versions { if obj.Name >= p.KeyMarker && obj.Version() >= p.VersionIDMarker {
sortedNames = append(sortedNames, k) allObjects = allObjects[i:]
}
sort.Strings(sortedNames)
objects := make([]*ObjectVersionInfo, 0, p.MaxKeys)
for _, name := range sortedNames {
objects = append(objects, versions[name].getFiltered()...)
if len(objects) > p.MaxKeys {
objects = objects[:p.MaxKeys]
break break
} }
} }
res.CommonPrefixes, allObjects = triageObjects(allObjects)
if len(allObjects) > p.MaxKeys {
res.IsTruncated = true
res.NextKeyMarker = allObjects[p.MaxKeys].Name
res.NextVersionIDMarker = allObjects[p.MaxKeys].Version()
allObjects = allObjects[:p.MaxKeys]
res.KeyMarker = allObjects[p.MaxKeys-1].Name
res.VersionIDMarker = allObjects[p.MaxKeys-1].Version()
}
objects := make([]*ObjectVersionInfo, len(allObjects))
for i, obj := range allObjects {
objects[i] = &ObjectVersionInfo{Object: obj}
if i == len(allObjects)-1 || allObjects[i+1].Name != obj.Name {
objects[i].IsLatest = true
}
}
res.Version, res.DeleteMarker = triageVersions(objects) res.Version, res.DeleteMarker = triageVersions(objects)
return &res, nil return res, nil
} }
func sortVersions(versions []*ObjectInfo) { func sortVersions(versions []*ObjectInfo) {
@ -773,7 +775,7 @@ func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*Bu
return n.getBucketSettings(ctx, bktInfo) return n.getBucketSettings(ctx, bktInfo)
} }
func (n *layer) getBucketSettings(ctx context.Context, bktInfo *BucketInfo) (*BucketSettings, error) { func (n *layer) getBucketSettings(ctx context.Context, bktInfo *cache.BucketInfo) (*BucketSettings, error) {
objInfo, err := n.getSettingsObjectInfo(ctx, bktInfo) objInfo, err := n.getSettingsObjectInfo(ctx, bktInfo)
if err != nil { if err != nil {
return nil, err return nil, err

View file

@ -14,6 +14,8 @@ import (
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors" apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -29,9 +31,10 @@ type (
io.Writer io.Writer
*object.Range *object.Range
offset int64 offset int64
length int64 length int64
address *object.Address cid *cid.ID
oid *object.ID
} }
// ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2. // ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2.
@ -58,7 +61,7 @@ type (
} }
allObjectParams struct { allObjectParams struct {
Bucket *BucketInfo Bucket *cache.BucketInfo
Delimiter string Delimiter string
Prefix string Prefix string
} }
@ -96,12 +99,16 @@ func (n *layer) objectFindID(ctx context.Context, p *findParams) (*object.ID, er
return nil, errors.New("several objects with the same name found") return nil, errors.New("several objects with the same name found")
} }
// objectHead returns all object's headers. func newAddress(cid *cid.ID, oid *object.ID) *object.Address {
func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*object.Object, error) {
address := object.NewAddress() address := object.NewAddress()
address.SetContainerID(cid) address.SetContainerID(cid)
address.SetObjectID(oid) address.SetObjectID(oid)
ops := new(client.ObjectHeaderParams).WithAddress(address).WithAllFields() return address
}
// objectHead returns all object's headers.
func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*object.Object, error) {
ops := new(client.ObjectHeaderParams).WithAddress(newAddress(cid, oid)).WithAllFields()
return n.pool.GetObjectHeader(ctx, ops, n.BearerOpt(ctx)) return n.pool.GetObjectHeader(ctx, ops, n.BearerOpt(ctx))
} }
@ -109,19 +116,19 @@ func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*o
func (n *layer) objectGet(ctx context.Context, p *getParams) (*object.Object, error) { func (n *layer) objectGet(ctx context.Context, p *getParams) (*object.Object, error) {
// prepare length/offset writer // prepare length/offset writer
w := newWriter(p.Writer, p.offset, p.length) w := newWriter(p.Writer, p.offset, p.length)
ops := new(client.GetObjectParams).WithAddress(p.address).WithPayloadWriter(w) ops := new(client.GetObjectParams).WithAddress(newAddress(p.cid, p.oid)).WithPayloadWriter(w)
return n.pool.GetObject(ctx, ops, n.BearerOpt(ctx)) return n.pool.GetObject(ctx, ops, n.BearerOpt(ctx))
} }
// objectRange gets object range and writes it into provided io.Writer. // objectRange gets object range and writes it into provided io.Writer.
func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) { func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) {
w := newWriter(p.Writer, p.offset, p.length) w := newWriter(p.Writer, p.offset, p.length)
ops := new(client.RangeDataParams).WithAddress(p.address).WithDataWriter(w).WithRange(p.Range) ops := new(client.RangeDataParams).WithAddress(newAddress(p.cid, p.oid)).WithDataWriter(w).WithRange(p.Range)
return n.pool.ObjectPayloadRangeData(ctx, ops, n.BearerOpt(ctx)) return n.pool.ObjectPayloadRangeData(ctx, ops, n.BearerOpt(ctx))
} }
// objectPut into NeoFS, took payload from io.Reader. // objectPut into NeoFS, took payload from io.Reader.
func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectParams) (*ObjectInfo, error) { func (n *layer) objectPut(ctx context.Context, bkt *cache.BucketInfo, p *PutObjectParams) (*ObjectInfo, error) {
own := n.Owner(ctx) own := n.Owner(ctx)
obj, err := url.QueryUnescape(p.Object) obj, err := url.QueryUnescape(p.Object)
if err != nil { if err != nil {
@ -135,8 +142,15 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara
} }
idsToDeleteArr := updateCRDT2PSetHeaders(p, versions, versioningEnabled) idsToDeleteArr := updateCRDT2PSetHeaders(p, versions, versioningEnabled)
r := p.Reader
if len(p.Header[api.ContentType]) == 0 {
d := newDetector(p.Reader)
if contentType, err := d.Detect(); err == nil {
p.Header[api.ContentType] = contentType
}
r = d.MultiReader()
}
rawObject := formRawObject(p, bkt.CID, own, obj) rawObject := formRawObject(p, bkt.CID, own, obj)
r := newDetector(p.Reader)
ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(r) ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(r)
oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx)) oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx))
@ -144,11 +158,21 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara
return nil, err return nil, err
} }
if p.Header[versionsDeleteMarkAttr] == delMarkFullObject {
if last := versions.getLast(); last != nil {
n.objCache.Delete(last.Address())
}
}
meta, err := n.objectHead(ctx, bkt.CID, oid) meta, err := n.objectHead(ctx, bkt.CID, oid)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err = n.objCache.Put(*meta); err != nil {
n.log.Error("couldn't cache an object", zap.Error(err))
}
for _, id := range idsToDeleteArr { for _, id := range idsToDeleteArr {
if err = n.objectDelete(ctx, bkt.CID, id); err != nil { if err = n.objectDelete(ctx, bkt.CID, id); err != nil {
n.log.Warn("couldn't delete object", n.log.Warn("couldn't delete object",
@ -168,7 +192,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara
Created: time.Now(), Created: time.Now(),
CreationEpoch: meta.CreationEpoch(), CreationEpoch: meta.CreationEpoch(),
Headers: p.Header, Headers: p.Header,
ContentType: r.contentType, ContentType: p.Header[api.ContentType],
HashSum: meta.PayloadChecksum().String(), HashSum: meta.PayloadChecksum().String(),
}, nil }, nil
} }
@ -229,9 +253,12 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio
versionsDeletedStr += "," versionsDeletedStr += ","
} }
lastVersion := versions.getLast() if lastVersion := versions.getLast(); lastVersion != nil {
p.Header[versionsDelAttr] = versionsDeletedStr + lastVersion.Version() p.Header[versionsDelAttr] = versionsDeletedStr + lastVersion.Version()
idsToDeleteArr = append(idsToDeleteArr, lastVersion.ID()) idsToDeleteArr = append(idsToDeleteArr, lastVersion.ID())
} else if len(versionsDeletedStr) != 0 {
p.Header[versionsDelAttr] = versionsDeletedStr
}
for _, version := range versions.objects { for _, version := range versions.objects {
if contains(versions.delList, version.Version()) { if contains(versions.delList, version.Version()) {
@ -243,7 +270,13 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio
return idsToDeleteArr return idsToDeleteArr
} }
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *cache.BucketInfo, objectName string) (*ObjectInfo, error) {
if address := n.headCache.Get(bkt.Name + "/" + objectName); address != nil {
if headInfo := n.objCache.Get(address); headInfo != nil {
return objInfoFromMeta(bkt, headInfo), nil
}
}
versions, err := n.headVersions(ctx, bkt, objectName) versions, err := n.headVersions(ctx, bkt, objectName)
if err != nil { if err != nil {
return nil, err return nil, err
@ -253,10 +286,17 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *BucketInfo
if lastVersion == nil { if lastVersion == nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
} }
if err = n.headCache.Put(lastVersion.NiceName(), lastVersion.Address()); err != nil {
n.log.Warn("couldn't put obj address to head cache",
zap.String("obj nice name", lastVersion.NiceName()),
zap.Error(err))
}
return lastVersion, nil return lastVersion, nil
} }
func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName string) (*objectVersions, error) { func (n *layer) headVersions(ctx context.Context, bkt *cache.BucketInfo, objectName string) (*objectVersions, error) {
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName}) ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName})
if err != nil { if err != nil {
return nil, err return nil, err
@ -276,6 +316,13 @@ func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName st
zap.Error(err)) zap.Error(err))
continue continue
} }
if err = n.objCache.Put(*meta); err != nil {
n.log.Warn("couldn't put meta to objects cache",
zap.Stringer("object id", id),
zap.Stringer("bucket id", bkt.CID),
zap.Error(err))
}
if oi := objectInfoFromMeta(bkt, meta, "", ""); oi != nil { if oi := objectInfoFromMeta(bkt, meta, "", ""); oi != nil {
if isSystem(oi) { if isSystem(oi) {
continue continue
@ -287,12 +334,16 @@ func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName st
return versions, nil return versions, nil
} }
func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID string) (*ObjectInfo, error) { func (n *layer) headVersion(ctx context.Context, bkt *cache.BucketInfo, versionID string) (*ObjectInfo, error) {
oid := object.NewID() oid := object.NewID()
if err := oid.Parse(versionID); err != nil { if err := oid.Parse(versionID); err != nil {
return nil, err return nil, err
} }
if headInfo := n.objCache.Get(newAddress(bkt.CID, oid)); headInfo != nil {
return objInfoFromMeta(bkt, headInfo), nil
}
meta, err := n.objectHead(ctx, bkt.CID, oid) meta, err := n.objectHead(ctx, bkt.CID, oid)
if err != nil { if err != nil {
if strings.Contains(err.Error(), "not found") { if strings.Contains(err.Error(), "not found") {
@ -301,14 +352,22 @@ func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID stri
return nil, err return nil, err
} }
return objectInfoFromMeta(bkt, meta, "", ""), nil objInfo := objectInfoFromMeta(bkt, meta, "", "")
if err = n.objCache.Put(*meta); err != nil {
n.log.Warn("couldn't put obj to object cache",
zap.String("bucket name", objInfo.Bucket),
zap.Stringer("bucket cid", objInfo.CID()),
zap.String("object name", objInfo.Name),
zap.Stringer("object id", objInfo.ID()),
zap.Error(err))
}
return objInfo, nil
} }
// objectDelete puts tombstone object into neofs. // objectDelete puts tombstone object into neofs.
func (n *layer) objectDelete(ctx context.Context, cid *cid.ID, oid *object.ID) error { func (n *layer) objectDelete(ctx context.Context, cid *cid.ID, oid *object.ID) error {
address := object.NewAddress() address := newAddress(cid, oid)
address.SetContainerID(cid)
address.SetObjectID(oid)
dop := new(client.DeleteObjectParams) dop := new(client.DeleteObjectParams)
dop.WithAddress(address) dop.WithAddress(address)
n.objCache.Delete(address) n.objCache.Delete(address)
@ -390,35 +449,11 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
} }
func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) { func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) {
ids, err := n.objectSearch(ctx, &findParams{cid: p.Bucket.CID}) versions, err := n.getAllObjectsVersions(ctx, p.Bucket, p.Prefix, p.Delimiter)
if err != nil { if err != nil {
return nil, err return nil, err
} }
versions := make(map[string]*objectVersions, len(ids)/2)
for _, id := range ids {
meta, err := n.objectHead(ctx, p.Bucket.CID, id)
if err != nil {
n.log.Warn("could not fetch object meta", zap.Error(err))
continue
}
if oi := objectInfoFromMeta(p.Bucket, meta, p.Prefix, p.Delimiter); oi != nil {
if isSystem(oi) {
continue
}
if objVersions, ok := versions[oi.Name]; ok {
objVersions.appendVersion(oi)
versions[oi.Name] = objVersions
} else {
objVersion := newObjectVersions(oi.Name)
objVersion.appendVersion(oi)
versions[oi.Name] = objVersion
}
}
}
objects := make([]*ObjectInfo, 0, len(versions)) objects := make([]*ObjectInfo, 0, len(versions))
for _, v := range versions { for _, v := range versions {
lastVersion := v.getLast() lastVersion := v.getLast()
@ -434,6 +469,38 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam
return objects, nil return objects, nil
} }
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *cache.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) {
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID})
if err != nil {
return nil, err
}
versions := make(map[string]*objectVersions, len(ids)/2)
for _, id := range ids {
meta, err := n.objectHead(ctx, bkt.CID, id)
if err != nil {
n.log.Warn("could not fetch object meta", zap.Error(err))
continue
}
if oi := objectInfoFromMeta(bkt, meta, prefix, delimiter); oi != nil {
if isSystem(oi) {
continue
}
if objVersions, ok := versions[oi.Name]; ok {
objVersions.appendVersion(oi)
versions[oi.Name] = objVersions
} else {
objVersion := newObjectVersions(oi.Name)
objVersion.appendVersion(oi)
versions[oi.Name] = objVersion
}
}
}
return versions, nil
}
func getExistedVersions(versions *objectVersions) []string { func getExistedVersions(versions *objectVersions) []string {
var res []string var res []string
for _, add := range versions.addList { for _, add := range versions.addList {
@ -498,7 +565,7 @@ func triageObjects(allObjects []*ObjectInfo) (prefixes []string, objects []*Obje
func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*ObjectInfo, error) { func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*ObjectInfo, error) {
var ( var (
err error err error
bkt *BucketInfo bkt *cache.BucketInfo
cacheKey cacheOptions cacheKey cacheOptions
allObjects []*ObjectInfo allObjects []*ObjectInfo
) )
@ -507,11 +574,11 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
return nil, err return nil, err
} }
if cacheKey, err = createKey(ctx, bkt.CID, p.Prefix, p.Delimiter); err != nil { if cacheKey, err = createKey(ctx, bkt.CID, listObjectsMethod, p.Prefix, p.Delimiter); err != nil {
return nil, err return nil, err
} }
allObjects = n.listObjCache.Get(cacheKey) allObjects = n.listsCache.Get(cacheKey)
if allObjects == nil { if allObjects == nil {
allObjects, err = n.listSortedObjectsFromNeoFS(ctx, allObjectParams{ allObjects, err = n.listSortedObjectsFromNeoFS(ctx, allObjectParams{
@ -524,13 +591,13 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
} }
// putting to cache a copy of allObjects because allObjects can be modified further // putting to cache a copy of allObjects because allObjects can be modified further
n.listObjCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...)) n.listsCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...))
} }
return allObjects, nil return allObjects, nil
} }
func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *BucketInfo) bool { func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *cache.BucketInfo) bool {
settings, err := n.getBucketSettings(ctx, bktInfo) settings, err := n.getBucketSettings(ctx, bktInfo)
if err != nil { if err != nil {
n.log.Warn("couldn't get versioning settings object", zap.Error(err)) n.log.Warn("couldn't get versioning settings object", zap.Error(err))

View file

@ -30,6 +30,11 @@ type (
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects. // DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
const DefaultObjectsListCacheLifetime = time.Second * 60 const DefaultObjectsListCacheLifetime = time.Second * 60
const (
listObjectsMethod = "listObjects"
listVersionsMethod = "listVersions"
)
type ( type (
listObjectsCache struct { listObjectsCache struct {
cacheLifetime time.Duration cacheLifetime time.Duration
@ -78,7 +83,7 @@ func (l *listObjectsCache) Put(key cacheOptions, objects []*ObjectInfo) {
}) })
} }
func createKey(ctx context.Context, cid *cid.ID, prefix, delimiter string) (cacheOptions, error) { func createKey(ctx context.Context, cid *cid.ID, method, prefix, delimiter string) (cacheOptions, error) {
box, err := GetBoxData(ctx) box, err := GetBoxData(ctx)
if err != nil { if err != nil {
return cacheOptions{}, err return cacheOptions{}, err

View file

@ -12,6 +12,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
) )
@ -60,7 +61,7 @@ type (
// ListObjectVersionsInfo stores info and list of objects' versions. // ListObjectVersionsInfo stores info and list of objects' versions.
ListObjectVersionsInfo struct { ListObjectVersionsInfo struct {
CommonPrefixes []*string CommonPrefixes []string
IsTruncated bool IsTruncated bool
KeyMarker string KeyMarker string
NextKeyMarker string NextKeyMarker string
@ -84,7 +85,11 @@ func userHeaders(attrs []*object.Attribute) map[string]string {
return result return result
} }
func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo { func objInfoFromMeta(bkt *cache.BucketInfo, meta *object.Object) *ObjectInfo {
return objectInfoFromMeta(bkt, meta, "", "")
}
func objectInfoFromMeta(bkt *cache.BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo {
var ( var (
isDir bool isDir bool
size int64 size int64
@ -164,6 +169,12 @@ func (o *ObjectInfo) ID() *object.ID { return o.id }
// Version returns object version from ObjectInfo. // Version returns object version from ObjectInfo.
func (o *ObjectInfo) Version() string { return o.id.String() } func (o *ObjectInfo) Version() string { return o.id.String() }
// NiceName returns object name for cache.
func (o *ObjectInfo) NiceName() string { return o.Bucket + "/" + o.Name }
// Address returns object address.
func (o *ObjectInfo) Address() *object.Address { return newAddress(o.bucketID, o.id) }
// CID returns bucket ID from ObjectInfo. // CID returns bucket ID from ObjectInfo.
func (o *ObjectInfo) CID() *cid.ID { return o.bucketID } func (o *ObjectInfo) CID() *cid.ID { return o.bucketID }

View file

@ -9,6 +9,7 @@ import (
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -19,7 +20,7 @@ var (
defaultTestContentType = http.DetectContentType(defaultTestPayload) defaultTestContentType = http.DetectContentType(defaultTestPayload)
) )
func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object { func newTestObject(oid *object.ID, bkt *cache.BucketInfo, name string) *object.Object {
filename := object.NewAttribute() filename := object.NewAttribute()
filename.SetKey(object.AttributeFileName) filename.SetKey(object.AttributeFileName)
filename.SetValue(name) filename.SetValue(name)
@ -43,7 +44,7 @@ func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object
return raw.Object() return raw.Object()
} }
func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *ObjectInfo { func newTestInfo(oid *object.ID, bkt *cache.BucketInfo, name string, isDir bool) *ObjectInfo {
info := &ObjectInfo{ info := &ObjectInfo{
id: oid, id: oid,
Name: name, Name: name,
@ -71,7 +72,7 @@ func Test_objectInfoFromMeta(t *testing.T) {
oid := object.NewID() oid := object.NewID()
containerID := cid.New() containerID := cid.New()
bkt := &BucketInfo{ bkt := &cache.BucketInfo{
Name: "test-container", Name: "test-container",
CID: containerID, CID: containerID,
Owner: uid, Owner: uid,

View file

@ -20,6 +20,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/session" "github.com/nspcc-dev/neofs-api-go/pkg/session"
"github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-api-go/pkg/token"
"github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/pkg/logger" "github.com/nspcc-dev/neofs-sdk-go/pkg/logger"
"github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool"
@ -61,10 +62,7 @@ func (t *testPool) PutObject(ctx context.Context, params *client.PutObjectParams
raw.SetPayload(all) raw.SetPayload(all)
} }
addr := object.NewAddress() addr := newAddress(raw.ContainerID(), raw.ID())
addr.SetObjectID(raw.ID())
addr.SetContainerID(raw.ContainerID())
t.objects[addr.String()] = raw.Object() t.objects[addr.String()] = raw.Object()
return raw.ID(), nil return raw.ID(), nil
} }
@ -329,8 +327,12 @@ func prepareContext(t *testing.T) *testContext {
require.NoError(t, err) require.NoError(t, err)
return &testContext{ return &testContext{
ctx: ctx, ctx: ctx,
layer: NewLayer(l, tp), layer: NewLayer(l, tp, &CacheConfig{
Size: cache.DefaultObjectsCacheSize,
Lifetime: cache.DefaultObjectsCacheLifetime,
ListObjectsLifetime: DefaultObjectsListCacheLifetime},
),
bkt: bktName, bkt: bktName,
bktID: bktID, bktID: bktID,
obj: "obj1", obj: "obj1",