forked from TrueCloudLab/frostfs-s3-gw
[#122] Add versioning cache
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
parent
f6c51cc9ee
commit
11558124cd
15 changed files with 503 additions and 203 deletions
64
api/cache/bucket.go
vendored
Normal file
64
api/cache/bucket.go
vendored
Normal file
|
@ -0,0 +1,64 @@
|
||||||
|
package cache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/bluele/gcache"
|
||||||
|
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
||||||
|
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
// BucketCache provides interface for lru cache for objects.
|
||||||
|
BucketCache interface {
|
||||||
|
Get(key string) *BucketInfo
|
||||||
|
Put(bkt *BucketInfo) error
|
||||||
|
Delete(key string) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// BucketInfo stores basic bucket data.
|
||||||
|
BucketInfo struct {
|
||||||
|
Name string
|
||||||
|
CID *cid.ID
|
||||||
|
Owner *owner.ID
|
||||||
|
Created time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBucketCache contains cache with objects and lifetime of cache entries.
|
||||||
|
GetBucketCache struct {
|
||||||
|
cache gcache.Cache
|
||||||
|
lifetime time.Duration
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewBucketCache creates an object of BucketCache.
|
||||||
|
func NewBucketCache(cacheSize int, lifetime time.Duration) *GetBucketCache {
|
||||||
|
gc := gcache.New(cacheSize).LRU().Build()
|
||||||
|
|
||||||
|
return &GetBucketCache{cache: gc, lifetime: lifetime}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns cached object.
|
||||||
|
func (o *GetBucketCache) Get(key string) *BucketInfo {
|
||||||
|
entry, err := o.cache.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, ok := entry.(*BucketInfo)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put puts an object to cache.
|
||||||
|
func (o *GetBucketCache) Put(bkt *BucketInfo) error {
|
||||||
|
return o.cache.SetWithExpire(bkt.Name, bkt, o.lifetime)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete deletes an object from cache.
|
||||||
|
func (o *GetBucketCache) Delete(key string) bool {
|
||||||
|
return o.cache.Remove(key)
|
||||||
|
}
|
55
api/cache/head_cache.go
vendored
Normal file
55
api/cache/head_cache.go
vendored
Normal file
|
@ -0,0 +1,55 @@
|
||||||
|
package cache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/bluele/gcache"
|
||||||
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HeadObjectsCache provides interface for lru cache for objects.
|
||||||
|
type HeadObjectsCache interface {
|
||||||
|
Get(key string) *object.Address
|
||||||
|
Put(key string, address *object.Address) error
|
||||||
|
Delete(key string) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type (
|
||||||
|
// HeadObjectCache contains cache with objects and lifetime of cache entries.
|
||||||
|
HeadObjectCache struct {
|
||||||
|
cache gcache.Cache
|
||||||
|
lifetime time.Duration
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewHeadObject creates an object of ObjectHeadersCache.
|
||||||
|
func NewHeadObject(cacheSize int, lifetime time.Duration) *HeadObjectCache {
|
||||||
|
gc := gcache.New(cacheSize).LRU().Build()
|
||||||
|
|
||||||
|
return &HeadObjectCache{cache: gc, lifetime: lifetime}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns cached object.
|
||||||
|
func (o *HeadObjectCache) Get(key string) *object.Address {
|
||||||
|
entry, err := o.cache.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, ok := entry.(*object.Address)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put puts an object to cache.
|
||||||
|
func (o *HeadObjectCache) Put(key string, address *object.Address) error {
|
||||||
|
return o.cache.SetWithExpire(key, address, o.lifetime)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete deletes an object from cache.
|
||||||
|
func (o *HeadObjectCache) Delete(key string) bool {
|
||||||
|
return o.cache.Remove(key)
|
||||||
|
}
|
6
api/cache/object_cache.go
vendored
6
api/cache/object_cache.go
vendored
|
@ -10,7 +10,7 @@ import (
|
||||||
// ObjectsCache provides interface for lru cache for objects.
|
// ObjectsCache provides interface for lru cache for objects.
|
||||||
type ObjectsCache interface {
|
type ObjectsCache interface {
|
||||||
Get(address *object.Address) *object.Object
|
Get(address *object.Address) *object.Object
|
||||||
Put(address *object.Address, obj object.Object) error
|
Put(obj object.Object) error
|
||||||
Delete(address *object.Address) bool
|
Delete(address *object.Address) bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -52,8 +52,8 @@ func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put puts an object to cache.
|
// Put puts an object to cache.
|
||||||
func (o *ObjectHeadersCache) Put(address *object.Address, obj object.Object) error {
|
func (o *ObjectHeadersCache) Put(obj object.Object) error {
|
||||||
return o.cache.SetWithExpire(address.String(), obj, o.lifetime)
|
return o.cache.SetWithExpire(obj.ContainerID().String()+"/"+obj.ID().String(), obj, o.lifetime)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete deletes an object from cache.
|
// Delete deletes an object from cache.
|
||||||
|
|
16
api/cache/object_cache_test.go
vendored
16
api/cache/object_cache_test.go
vendored
|
@ -4,6 +4,8 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
|
|
||||||
objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test"
|
objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -14,23 +16,23 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCache(t *testing.T) {
|
func TestCache(t *testing.T) {
|
||||||
var (
|
obj := objecttest.Object()
|
||||||
address = objecttest.Address()
|
address := object.NewAddress()
|
||||||
object = objecttest.Object()
|
address.SetContainerID(obj.ContainerID())
|
||||||
)
|
address.SetObjectID(obj.ID())
|
||||||
|
|
||||||
t.Run("check get", func(t *testing.T) {
|
t.Run("check get", func(t *testing.T) {
|
||||||
cache := New(cachesize, lifetime)
|
cache := New(cachesize, lifetime)
|
||||||
err := cache.Put(address, *object)
|
err := cache.Put(*obj)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
actual := cache.Get(address)
|
actual := cache.Get(address)
|
||||||
require.Equal(t, object, actual)
|
require.Equal(t, obj, actual)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("check delete", func(t *testing.T) {
|
t.Run("check delete", func(t *testing.T) {
|
||||||
cache := New(cachesize, lifetime)
|
cache := New(cachesize, lifetime)
|
||||||
err := cache.Put(address, *object)
|
err := cache.Put(*obj)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
cache.Delete(address)
|
cache.Delete(address)
|
||||||
|
|
56
api/cache/system.go
vendored
Normal file
56
api/cache/system.go
vendored
Normal file
|
@ -0,0 +1,56 @@
|
||||||
|
package cache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
|
|
||||||
|
"github.com/bluele/gcache"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
// SystemCache provides interface for lru cache for objects.
|
||||||
|
SystemCache interface {
|
||||||
|
Get(key string) *object.Object
|
||||||
|
Put(key string, obj *object.Object) error
|
||||||
|
Delete(key string) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// systemCache contains cache with objects and lifetime of cache entries.
|
||||||
|
systemCache struct {
|
||||||
|
cache gcache.Cache
|
||||||
|
lifetime time.Duration
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewSystemCache creates an object of SystemCache.
|
||||||
|
func NewSystemCache(cacheSize int, lifetime time.Duration) SystemCache {
|
||||||
|
gc := gcache.New(cacheSize).LRU().Build()
|
||||||
|
|
||||||
|
return &systemCache{cache: gc, lifetime: lifetime}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns cached object.
|
||||||
|
func (o *systemCache) Get(key string) *object.Object {
|
||||||
|
entry, err := o.cache.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, ok := entry.(*object.Object)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put puts an object to cache.
|
||||||
|
func (o *systemCache) Put(key string, obj *object.Object) error {
|
||||||
|
return o.cache.SetWithExpire(key, obj, o.lifetime)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete deletes an object from cache.
|
||||||
|
func (o *systemCache) Delete(key string) bool {
|
||||||
|
return o.cache.Remove(key)
|
||||||
|
}
|
|
@ -46,18 +46,22 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "could not fetch object info", reqInfo, err)
|
h.logAndSendError(w, "could not fetch object info", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType))
|
|
||||||
getParams := &layer.GetObjectParams{
|
if len(inf.ContentType) == 0 {
|
||||||
ObjectInfo: inf,
|
buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType))
|
||||||
Writer: buffer,
|
getParams := &layer.GetObjectParams{
|
||||||
Range: getRangeToDetectContentType(inf.Size),
|
ObjectInfo: inf,
|
||||||
VersionID: reqInfo.URL.Query().Get("versionId"),
|
Writer: buffer,
|
||||||
|
Range: getRangeToDetectContentType(inf.Size),
|
||||||
|
VersionID: reqInfo.URL.Query().Get("versionId"),
|
||||||
|
}
|
||||||
|
if err = h.obj.GetObject(r.Context(), getParams); err != nil {
|
||||||
|
h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", inf.ID()))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
inf.ContentType = http.DetectContentType(buffer.Bytes())
|
||||||
}
|
}
|
||||||
if err = h.obj.GetObject(r.Context(), getParams); err != nil {
|
|
||||||
h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", inf.ID()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
inf.ContentType = http.DetectContentType(buffer.Bytes())
|
|
||||||
writeHeaders(w.Header(), inf)
|
writeHeaders(w.Header(), inf)
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
|
@ -263,7 +263,7 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, prefix := range info.CommonPrefixes {
|
for _, prefix := range info.CommonPrefixes {
|
||||||
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix})
|
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: prefix})
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ver := range info.Version {
|
for _, ver := range info.Version {
|
||||||
|
|
|
@ -11,37 +11,29 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
|
"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/container"
|
"github.com/nspcc-dev/neofs-api-go/pkg/container"
|
||||||
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pkg/pool"
|
"github.com/nspcc-dev/neofs-sdk-go/pkg/pool"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
// BucketInfo stores basic bucket data.
|
|
||||||
BucketInfo struct {
|
|
||||||
Name string
|
|
||||||
CID *cid.ID
|
|
||||||
Owner *owner.ID
|
|
||||||
Created time.Time
|
|
||||||
BasicACL uint32
|
|
||||||
}
|
|
||||||
// BucketACL extends BucketInfo by eacl.Table.
|
// BucketACL extends BucketInfo by eacl.Table.
|
||||||
BucketACL struct {
|
BucketACL struct {
|
||||||
Info *BucketInfo
|
Info *cache.BucketInfo
|
||||||
EACL *eacl.Table
|
EACL *eacl.Table
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*BucketInfo, error) {
|
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*cache.BucketInfo, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
res *container.Container
|
res *container.Container
|
||||||
rid = api.GetRequestID(ctx)
|
rid = api.GetRequestID(ctx)
|
||||||
bearerOpt = n.BearerOpt(ctx)
|
bearerOpt = n.BearerOpt(ctx)
|
||||||
|
|
||||||
info = &BucketInfo{
|
info = &cache.BucketInfo{
|
||||||
CID: cid,
|
CID: cid,
|
||||||
Name: cid.String(),
|
Name: cid.String(),
|
||||||
}
|
}
|
||||||
|
@ -82,10 +74,17 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*BucketInfo, er
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := n.bucketCache.Put(info); err != nil {
|
||||||
|
n.log.Warn("could not put bucket info into cache",
|
||||||
|
zap.Stringer("cid", cid),
|
||||||
|
zap.String("bucket_name", info.Name),
|
||||||
|
zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) containerList(ctx context.Context) ([]*BucketInfo, error) {
|
func (n *layer) containerList(ctx context.Context) ([]*cache.BucketInfo, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
own = n.Owner(ctx)
|
own = n.Owner(ctx)
|
||||||
|
@ -101,7 +100,7 @@ func (n *layer) containerList(ctx context.Context) ([]*BucketInfo, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
list := make([]*BucketInfo, 0, len(res))
|
list := make([]*cache.BucketInfo, 0, len(res))
|
||||||
for _, cid := range res {
|
for _, cid := range res {
|
||||||
info, err := n.containerInfo(ctx, cid)
|
info, err := n.containerInfo(ctx, cid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -3,24 +3,56 @@ package layer
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type detector struct {
|
type (
|
||||||
io.Reader
|
detector struct {
|
||||||
sync.Once
|
io.Reader
|
||||||
|
err error
|
||||||
|
data []byte
|
||||||
|
}
|
||||||
|
errReader struct {
|
||||||
|
data []byte
|
||||||
|
err error
|
||||||
|
offset int
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
contentType string
|
const contentTypeDetectSize = 512
|
||||||
|
|
||||||
|
func newReader(data []byte, err error) *errReader {
|
||||||
|
return &errReader{data: data, err: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newDetector(r io.Reader) *detector {
|
func (r *errReader) Read(b []byte) (int, error) {
|
||||||
return &detector{Reader: r}
|
if r.offset >= len(r.data) {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
n := copy(b, r.data[r.offset:])
|
||||||
|
r.offset += n
|
||||||
|
if r.offset >= len(r.data) {
|
||||||
|
return n, r.err
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *detector) Read(data []byte) (int, error) {
|
func newDetector(reader io.Reader) *detector {
|
||||||
d.Do(func() {
|
return &detector{
|
||||||
d.contentType = http.DetectContentType(data)
|
data: make([]byte, contentTypeDetectSize),
|
||||||
})
|
Reader: reader,
|
||||||
|
}
|
||||||
return d.Reader.Read(data)
|
}
|
||||||
|
|
||||||
|
func (d *detector) Detect() (string, error) {
|
||||||
|
n, err := d.Reader.Read(d.data)
|
||||||
|
if err != nil && err != io.EOF {
|
||||||
|
d.err = err
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
d.data = d.data[:n]
|
||||||
|
return http.DetectContentType(d.data), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *detector) MultiReader() io.Reader {
|
||||||
|
return io.MultiReader(newReader(d.data, d.err), d.Reader)
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,10 +28,13 @@ import (
|
||||||
|
|
||||||
type (
|
type (
|
||||||
layer struct {
|
layer struct {
|
||||||
pool pool.Pool
|
pool pool.Pool
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
listObjCache ObjectsListCache
|
listsCache ObjectsListCache
|
||||||
objCache cache.ObjectsCache
|
objCache cache.ObjectsCache
|
||||||
|
headCache cache.HeadObjectsCache
|
||||||
|
bucketCache cache.BucketCache
|
||||||
|
systemCache cache.SystemCache
|
||||||
}
|
}
|
||||||
|
|
||||||
// CacheConfig contains params for caches.
|
// CacheConfig contains params for caches.
|
||||||
|
@ -156,8 +159,8 @@ type (
|
||||||
PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error)
|
PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error)
|
||||||
GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error)
|
GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error)
|
||||||
|
|
||||||
ListBuckets(ctx context.Context) ([]*BucketInfo, error)
|
ListBuckets(ctx context.Context) ([]*cache.BucketInfo, error)
|
||||||
GetBucketInfo(ctx context.Context, name string) (*BucketInfo, error)
|
GetBucketInfo(ctx context.Context, name string) (*cache.BucketInfo, error)
|
||||||
GetBucketACL(ctx context.Context, name string) (*BucketACL, error)
|
GetBucketACL(ctx context.Context, name string) (*BucketACL, error)
|
||||||
PutBucketACL(ctx context.Context, p *PutBucketACLParams) error
|
PutBucketACL(ctx context.Context, p *PutBucketACLParams) error
|
||||||
CreateBucket(ctx context.Context, p *CreateBucketParams) (*cid.ID, error)
|
CreateBucket(ctx context.Context, p *CreateBucketParams) (*cid.ID, error)
|
||||||
|
@ -228,26 +231,22 @@ func (v *objectVersions) getLast() *ObjectInfo {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *objectVersions) getFiltered() []*ObjectVersionInfo {
|
func (v *objectVersions) getFiltered() []*ObjectInfo {
|
||||||
if len(v.objects) == 0 {
|
if len(v.objects) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
v.sort()
|
v.sort()
|
||||||
existedVersions := getExistedVersions(v)
|
existedVersions := getExistedVersions(v)
|
||||||
res := make([]*ObjectVersionInfo, 0, len(v.objects))
|
res := make([]*ObjectInfo, 0, len(v.objects))
|
||||||
|
|
||||||
for _, version := range v.objects {
|
for _, version := range v.objects {
|
||||||
delMark := version.Headers[versionsDeleteMarkAttr]
|
delMark := version.Headers[versionsDeleteMarkAttr]
|
||||||
if contains(existedVersions, version.Version()) && (delMark == delMarkFullObject || delMark == "") {
|
if contains(existedVersions, version.Version()) && (delMark == delMarkFullObject || delMark == "") {
|
||||||
res = append(res, &ObjectVersionInfo{Object: version})
|
res = append(res, version)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(res) > 0 {
|
|
||||||
res[len(res)-1].IsLatest = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -279,10 +278,14 @@ func (t *VersionedObject) String() string {
|
||||||
// and establishes gRPC connection with node.
|
// and establishes gRPC connection with node.
|
||||||
func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
|
func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
|
||||||
return &layer{
|
return &layer{
|
||||||
pool: conns,
|
pool: conns,
|
||||||
log: log,
|
log: log,
|
||||||
listObjCache: newListObjectsCache(config.ListObjectsLifetime),
|
listsCache: newListObjectsCache(config.ListObjectsLifetime),
|
||||||
objCache: cache.New(config.Size, config.Lifetime),
|
objCache: cache.New(config.Size, config.Lifetime),
|
||||||
|
//todo reconsider cache params
|
||||||
|
headCache: cache.NewHeadObject(1000, time.Minute),
|
||||||
|
bucketCache: cache.NewBucketCache(150, time.Minute),
|
||||||
|
systemCache: cache.NewSystemCache(1000, 5*time.Minute),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -320,12 +323,16 @@ func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Objec
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBucketInfo returns bucket info by name.
|
// GetBucketInfo returns bucket info by name.
|
||||||
func (n *layer) GetBucketInfo(ctx context.Context, name string) (*BucketInfo, error) {
|
func (n *layer) GetBucketInfo(ctx context.Context, name string) (*cache.BucketInfo, error) {
|
||||||
name, err := url.QueryUnescape(name)
|
name, err := url.QueryUnescape(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if bktInfo := n.bucketCache.Get(name); bktInfo != nil {
|
||||||
|
return bktInfo, nil
|
||||||
|
}
|
||||||
|
|
||||||
containerID := new(cid.ID)
|
containerID := new(cid.ID)
|
||||||
if err := containerID.Parse(name); err != nil {
|
if err := containerID.Parse(name); err != nil {
|
||||||
list, err := n.containerList(ctx)
|
list, err := n.containerList(ctx)
|
||||||
|
@ -374,7 +381,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err
|
||||||
|
|
||||||
// ListBuckets returns all user containers. Name of the bucket is a container
|
// ListBuckets returns all user containers. Name of the bucket is a container
|
||||||
// id. Timestamp is omitted since it is not saved in neofs container.
|
// id. Timestamp is omitted since it is not saved in neofs container.
|
||||||
func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) {
|
func (n *layer) ListBuckets(ctx context.Context) ([]*cache.BucketInfo, error) {
|
||||||
return n.containerList(ctx)
|
return n.containerList(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -382,21 +389,12 @@ func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) {
|
||||||
func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
//if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
|
|
||||||
// return fmt.Errorf("couldn't find bucket: %s : %w", p.Bucket, err)
|
|
||||||
//} else if oid, err = n.objectFindID(ctx, &findParams{cid: bkt.CID, val: p.Object}); err != nil {
|
|
||||||
// return fmt.Errorf("search of the object failed: cid: %s, val: %s : %w", bkt.CID, p.Object, err)
|
|
||||||
//}
|
|
||||||
|
|
||||||
addr := object.NewAddress()
|
|
||||||
addr.SetObjectID(p.ObjectInfo.ID())
|
|
||||||
addr.SetContainerID(p.ObjectInfo.CID())
|
|
||||||
|
|
||||||
params := &getParams{
|
params := &getParams{
|
||||||
Writer: p.Writer,
|
Writer: p.Writer,
|
||||||
address: addr,
|
cid: p.ObjectInfo.CID(),
|
||||||
offset: p.Offset,
|
oid: p.ObjectInfo.ID(),
|
||||||
length: p.Length,
|
offset: p.Offset,
|
||||||
|
length: p.Length,
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.Range != nil {
|
if p.Range != nil {
|
||||||
|
@ -411,7 +409,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.objCache.Delete(addr)
|
n.objCache.Delete(p.ObjectInfo.Address())
|
||||||
return fmt.Errorf("couldn't get object, cid: %s : %w", p.ObjectInfo.CID(), err)
|
return fmt.Errorf("couldn't get object, cid: %s : %w", p.ObjectInfo.CID(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -433,32 +431,26 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object
|
||||||
return n.headVersion(ctx, bkt, p.VersionID)
|
return n.headVersion(ctx, bkt, p.VersionID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) {
|
func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *cache.BucketInfo) (*ObjectInfo, error) {
|
||||||
|
if meta := n.systemCache.Get(bktVersionSettingsObject); meta != nil {
|
||||||
|
return objInfoFromMeta(bkt, meta), nil
|
||||||
|
}
|
||||||
|
|
||||||
oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bktVersionSettingsObject})
|
oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bktVersionSettingsObject})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
addr := object.NewAddress()
|
meta, err := n.objectHead(ctx, bkt.CID, oid)
|
||||||
addr.SetObjectID(oid)
|
if err != nil {
|
||||||
addr.SetContainerID(bkt.CID)
|
n.log.Error("could not fetch object head", zap.Error(err))
|
||||||
|
return nil, err
|
||||||
/* todo: now we get an address via request to NeoFS and try to find the object with the address in cache
|
}
|
||||||
but it will be resolved after implementation of local cache with nicenames and address of objects
|
if err = n.systemCache.Put(bktVersionSettingsObject, meta); err != nil {
|
||||||
for get/head requests */
|
n.log.Error("couldn't cache system object", zap.Error(err))
|
||||||
meta := n.objCache.Get(addr)
|
|
||||||
if meta == nil {
|
|
||||||
meta, err = n.objectHead(ctx, bkt.CID, oid)
|
|
||||||
if err != nil {
|
|
||||||
n.log.Error("could not fetch object head", zap.Error(err))
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err = n.objCache.Put(addr, *meta); err != nil {
|
|
||||||
n.log.Error("couldn't cache an object", zap.Error(err))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return objectInfoFromMeta(bkt, meta, "", ""), nil
|
return objInfoFromMeta(bkt, meta), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PutObject into storage.
|
// PutObject into storage.
|
||||||
|
@ -496,7 +488,7 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteObject removes all objects with passed nice name.
|
// DeleteObject removes all objects with passed nice name.
|
||||||
func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) error {
|
func (n *layer) deleteObject(ctx context.Context, bkt *cache.BucketInfo, obj *VersionedObject) error {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
ids []*object.ID
|
ids []*object.ID
|
||||||
|
@ -543,7 +535,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) checkVersionsExists(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) (*object.ID, error) {
|
func (n *layer) checkVersionsExists(ctx context.Context, bkt *cache.BucketInfo, obj *VersionedObject) (*object.ID, error) {
|
||||||
id := object.NewID()
|
id := object.NewID()
|
||||||
if err := id.Parse(obj.VersionID); err != nil {
|
if err := id.Parse(obj.VersionID); err != nil {
|
||||||
return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()}
|
return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()}
|
||||||
|
@ -608,60 +600,70 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||||
res := ListObjectVersionsInfo{}
|
var versions map[string]*objectVersions
|
||||||
versions := make(map[string]*objectVersions)
|
res := &ListObjectVersionsInfo{}
|
||||||
|
|
||||||
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
|
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID})
|
|
||||||
|
cacheKey, err := createKey(ctx, bkt.CID, listVersionsMethod, p.Prefix, p.Delimiter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, id := range ids {
|
allObjects := n.listsCache.Get(cacheKey)
|
||||||
meta, err := n.objectHead(ctx, bkt.CID, id)
|
if allObjects == nil {
|
||||||
|
versions, err = n.getAllObjectsVersions(ctx, bkt, p.Prefix, p.Delimiter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.log.Warn("could not fetch object meta", zap.Error(err))
|
return nil, err
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
|
|
||||||
if oi.Name <= p.KeyMarker {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if isSystem(oi) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if objVersions, ok := versions[oi.Name]; ok {
|
sortedNames := make([]string, 0, len(versions))
|
||||||
objVersions.appendVersion(oi)
|
for k := range versions {
|
||||||
versions[oi.Name] = objVersions
|
sortedNames = append(sortedNames, k)
|
||||||
} else {
|
|
||||||
objVersion := newObjectVersions(oi.Name)
|
|
||||||
objVersion.appendVersion(oi)
|
|
||||||
versions[oi.Name] = objVersion
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
sort.Strings(sortedNames)
|
||||||
|
|
||||||
|
allObjects = make([]*ObjectInfo, 0, p.MaxKeys)
|
||||||
|
for _, name := range sortedNames {
|
||||||
|
allObjects = append(allObjects, versions[name].getFiltered()...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// putting to cache a copy of allObjects because allObjects can be modified further
|
||||||
|
n.listsCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...))
|
||||||
}
|
}
|
||||||
|
|
||||||
sortedNames := make([]string, 0, len(versions))
|
for i, obj := range allObjects {
|
||||||
for k := range versions {
|
if obj.Name >= p.KeyMarker && obj.Version() >= p.VersionIDMarker {
|
||||||
sortedNames = append(sortedNames, k)
|
allObjects = allObjects[i:]
|
||||||
}
|
|
||||||
sort.Strings(sortedNames)
|
|
||||||
|
|
||||||
objects := make([]*ObjectVersionInfo, 0, p.MaxKeys)
|
|
||||||
for _, name := range sortedNames {
|
|
||||||
objects = append(objects, versions[name].getFiltered()...)
|
|
||||||
if len(objects) > p.MaxKeys {
|
|
||||||
objects = objects[:p.MaxKeys]
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
res.CommonPrefixes, allObjects = triageObjects(allObjects)
|
||||||
|
|
||||||
|
if len(allObjects) > p.MaxKeys {
|
||||||
|
res.IsTruncated = true
|
||||||
|
res.NextKeyMarker = allObjects[p.MaxKeys].Name
|
||||||
|
res.NextVersionIDMarker = allObjects[p.MaxKeys].Version()
|
||||||
|
|
||||||
|
allObjects = allObjects[:p.MaxKeys]
|
||||||
|
res.KeyMarker = allObjects[p.MaxKeys-1].Name
|
||||||
|
res.VersionIDMarker = allObjects[p.MaxKeys-1].Version()
|
||||||
|
}
|
||||||
|
|
||||||
|
objects := make([]*ObjectVersionInfo, len(allObjects))
|
||||||
|
for i, obj := range allObjects {
|
||||||
|
objects[i] = &ObjectVersionInfo{Object: obj}
|
||||||
|
if i == len(allObjects)-1 || allObjects[i+1].Name != obj.Name {
|
||||||
|
objects[i].IsLatest = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
res.Version, res.DeleteMarker = triageVersions(objects)
|
res.Version, res.DeleteMarker = triageVersions(objects)
|
||||||
return &res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func sortVersions(versions []*ObjectInfo) {
|
func sortVersions(versions []*ObjectInfo) {
|
||||||
|
@ -773,7 +775,7 @@ func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*Bu
|
||||||
return n.getBucketSettings(ctx, bktInfo)
|
return n.getBucketSettings(ctx, bktInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) getBucketSettings(ctx context.Context, bktInfo *BucketInfo) (*BucketSettings, error) {
|
func (n *layer) getBucketSettings(ctx context.Context, bktInfo *cache.BucketInfo) (*BucketSettings, error) {
|
||||||
objInfo, err := n.getSettingsObjectInfo(ctx, bktInfo)
|
objInfo, err := n.getSettingsObjectInfo(ctx, bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
|
@ -14,6 +14,8 @@ import (
|
||||||
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -29,9 +31,10 @@ type (
|
||||||
io.Writer
|
io.Writer
|
||||||
*object.Range
|
*object.Range
|
||||||
|
|
||||||
offset int64
|
offset int64
|
||||||
length int64
|
length int64
|
||||||
address *object.Address
|
cid *cid.ID
|
||||||
|
oid *object.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2.
|
// ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2.
|
||||||
|
@ -58,7 +61,7 @@ type (
|
||||||
}
|
}
|
||||||
|
|
||||||
allObjectParams struct {
|
allObjectParams struct {
|
||||||
Bucket *BucketInfo
|
Bucket *cache.BucketInfo
|
||||||
Delimiter string
|
Delimiter string
|
||||||
Prefix string
|
Prefix string
|
||||||
}
|
}
|
||||||
|
@ -96,12 +99,16 @@ func (n *layer) objectFindID(ctx context.Context, p *findParams) (*object.ID, er
|
||||||
return nil, errors.New("several objects with the same name found")
|
return nil, errors.New("several objects with the same name found")
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectHead returns all object's headers.
|
func newAddress(cid *cid.ID, oid *object.ID) *object.Address {
|
||||||
func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*object.Object, error) {
|
|
||||||
address := object.NewAddress()
|
address := object.NewAddress()
|
||||||
address.SetContainerID(cid)
|
address.SetContainerID(cid)
|
||||||
address.SetObjectID(oid)
|
address.SetObjectID(oid)
|
||||||
ops := new(client.ObjectHeaderParams).WithAddress(address).WithAllFields()
|
return address
|
||||||
|
}
|
||||||
|
|
||||||
|
// objectHead returns all object's headers.
|
||||||
|
func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*object.Object, error) {
|
||||||
|
ops := new(client.ObjectHeaderParams).WithAddress(newAddress(cid, oid)).WithAllFields()
|
||||||
return n.pool.GetObjectHeader(ctx, ops, n.BearerOpt(ctx))
|
return n.pool.GetObjectHeader(ctx, ops, n.BearerOpt(ctx))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -109,19 +116,19 @@ func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*o
|
||||||
func (n *layer) objectGet(ctx context.Context, p *getParams) (*object.Object, error) {
|
func (n *layer) objectGet(ctx context.Context, p *getParams) (*object.Object, error) {
|
||||||
// prepare length/offset writer
|
// prepare length/offset writer
|
||||||
w := newWriter(p.Writer, p.offset, p.length)
|
w := newWriter(p.Writer, p.offset, p.length)
|
||||||
ops := new(client.GetObjectParams).WithAddress(p.address).WithPayloadWriter(w)
|
ops := new(client.GetObjectParams).WithAddress(newAddress(p.cid, p.oid)).WithPayloadWriter(w)
|
||||||
return n.pool.GetObject(ctx, ops, n.BearerOpt(ctx))
|
return n.pool.GetObject(ctx, ops, n.BearerOpt(ctx))
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectRange gets object range and writes it into provided io.Writer.
|
// objectRange gets object range and writes it into provided io.Writer.
|
||||||
func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) {
|
func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) {
|
||||||
w := newWriter(p.Writer, p.offset, p.length)
|
w := newWriter(p.Writer, p.offset, p.length)
|
||||||
ops := new(client.RangeDataParams).WithAddress(p.address).WithDataWriter(w).WithRange(p.Range)
|
ops := new(client.RangeDataParams).WithAddress(newAddress(p.cid, p.oid)).WithDataWriter(w).WithRange(p.Range)
|
||||||
return n.pool.ObjectPayloadRangeData(ctx, ops, n.BearerOpt(ctx))
|
return n.pool.ObjectPayloadRangeData(ctx, ops, n.BearerOpt(ctx))
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectPut into NeoFS, took payload from io.Reader.
|
// objectPut into NeoFS, took payload from io.Reader.
|
||||||
func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectParams) (*ObjectInfo, error) {
|
func (n *layer) objectPut(ctx context.Context, bkt *cache.BucketInfo, p *PutObjectParams) (*ObjectInfo, error) {
|
||||||
own := n.Owner(ctx)
|
own := n.Owner(ctx)
|
||||||
obj, err := url.QueryUnescape(p.Object)
|
obj, err := url.QueryUnescape(p.Object)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -135,8 +142,15 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara
|
||||||
}
|
}
|
||||||
idsToDeleteArr := updateCRDT2PSetHeaders(p, versions, versioningEnabled)
|
idsToDeleteArr := updateCRDT2PSetHeaders(p, versions, versioningEnabled)
|
||||||
|
|
||||||
|
r := p.Reader
|
||||||
|
if len(p.Header[api.ContentType]) == 0 {
|
||||||
|
d := newDetector(p.Reader)
|
||||||
|
if contentType, err := d.Detect(); err == nil {
|
||||||
|
p.Header[api.ContentType] = contentType
|
||||||
|
}
|
||||||
|
r = d.MultiReader()
|
||||||
|
}
|
||||||
rawObject := formRawObject(p, bkt.CID, own, obj)
|
rawObject := formRawObject(p, bkt.CID, own, obj)
|
||||||
r := newDetector(p.Reader)
|
|
||||||
|
|
||||||
ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(r)
|
ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(r)
|
||||||
oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx))
|
oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx))
|
||||||
|
@ -144,11 +158,21 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if p.Header[versionsDeleteMarkAttr] == delMarkFullObject {
|
||||||
|
if last := versions.getLast(); last != nil {
|
||||||
|
n.objCache.Delete(last.Address())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
meta, err := n.objectHead(ctx, bkt.CID, oid)
|
meta, err := n.objectHead(ctx, bkt.CID, oid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err = n.objCache.Put(*meta); err != nil {
|
||||||
|
n.log.Error("couldn't cache an object", zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
for _, id := range idsToDeleteArr {
|
for _, id := range idsToDeleteArr {
|
||||||
if err = n.objectDelete(ctx, bkt.CID, id); err != nil {
|
if err = n.objectDelete(ctx, bkt.CID, id); err != nil {
|
||||||
n.log.Warn("couldn't delete object",
|
n.log.Warn("couldn't delete object",
|
||||||
|
@ -168,7 +192,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara
|
||||||
Created: time.Now(),
|
Created: time.Now(),
|
||||||
CreationEpoch: meta.CreationEpoch(),
|
CreationEpoch: meta.CreationEpoch(),
|
||||||
Headers: p.Header,
|
Headers: p.Header,
|
||||||
ContentType: r.contentType,
|
ContentType: p.Header[api.ContentType],
|
||||||
HashSum: meta.PayloadChecksum().String(),
|
HashSum: meta.PayloadChecksum().String(),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
@ -229,9 +253,12 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio
|
||||||
versionsDeletedStr += ","
|
versionsDeletedStr += ","
|
||||||
}
|
}
|
||||||
|
|
||||||
lastVersion := versions.getLast()
|
if lastVersion := versions.getLast(); lastVersion != nil {
|
||||||
p.Header[versionsDelAttr] = versionsDeletedStr + lastVersion.Version()
|
p.Header[versionsDelAttr] = versionsDeletedStr + lastVersion.Version()
|
||||||
idsToDeleteArr = append(idsToDeleteArr, lastVersion.ID())
|
idsToDeleteArr = append(idsToDeleteArr, lastVersion.ID())
|
||||||
|
} else if len(versionsDeletedStr) != 0 {
|
||||||
|
p.Header[versionsDelAttr] = versionsDeletedStr
|
||||||
|
}
|
||||||
|
|
||||||
for _, version := range versions.objects {
|
for _, version := range versions.objects {
|
||||||
if contains(versions.delList, version.Version()) {
|
if contains(versions.delList, version.Version()) {
|
||||||
|
@ -243,7 +270,13 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio
|
||||||
return idsToDeleteArr
|
return idsToDeleteArr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) {
|
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *cache.BucketInfo, objectName string) (*ObjectInfo, error) {
|
||||||
|
if address := n.headCache.Get(bkt.Name + "/" + objectName); address != nil {
|
||||||
|
if headInfo := n.objCache.Get(address); headInfo != nil {
|
||||||
|
return objInfoFromMeta(bkt, headInfo), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
versions, err := n.headVersions(ctx, bkt, objectName)
|
versions, err := n.headVersions(ctx, bkt, objectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -253,10 +286,17 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *BucketInfo
|
||||||
if lastVersion == nil {
|
if lastVersion == nil {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err = n.headCache.Put(lastVersion.NiceName(), lastVersion.Address()); err != nil {
|
||||||
|
n.log.Warn("couldn't put obj address to head cache",
|
||||||
|
zap.String("obj nice name", lastVersion.NiceName()),
|
||||||
|
zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
return lastVersion, nil
|
return lastVersion, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName string) (*objectVersions, error) {
|
func (n *layer) headVersions(ctx context.Context, bkt *cache.BucketInfo, objectName string) (*objectVersions, error) {
|
||||||
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName})
|
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -276,6 +316,13 @@ func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName st
|
||||||
zap.Error(err))
|
zap.Error(err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if err = n.objCache.Put(*meta); err != nil {
|
||||||
|
n.log.Warn("couldn't put meta to objects cache",
|
||||||
|
zap.Stringer("object id", id),
|
||||||
|
zap.Stringer("bucket id", bkt.CID),
|
||||||
|
zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
if oi := objectInfoFromMeta(bkt, meta, "", ""); oi != nil {
|
if oi := objectInfoFromMeta(bkt, meta, "", ""); oi != nil {
|
||||||
if isSystem(oi) {
|
if isSystem(oi) {
|
||||||
continue
|
continue
|
||||||
|
@ -287,12 +334,16 @@ func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName st
|
||||||
return versions, nil
|
return versions, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID string) (*ObjectInfo, error) {
|
func (n *layer) headVersion(ctx context.Context, bkt *cache.BucketInfo, versionID string) (*ObjectInfo, error) {
|
||||||
oid := object.NewID()
|
oid := object.NewID()
|
||||||
if err := oid.Parse(versionID); err != nil {
|
if err := oid.Parse(versionID); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if headInfo := n.objCache.Get(newAddress(bkt.CID, oid)); headInfo != nil {
|
||||||
|
return objInfoFromMeta(bkt, headInfo), nil
|
||||||
|
}
|
||||||
|
|
||||||
meta, err := n.objectHead(ctx, bkt.CID, oid)
|
meta, err := n.objectHead(ctx, bkt.CID, oid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.Contains(err.Error(), "not found") {
|
if strings.Contains(err.Error(), "not found") {
|
||||||
|
@ -301,14 +352,22 @@ func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID stri
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return objectInfoFromMeta(bkt, meta, "", ""), nil
|
objInfo := objectInfoFromMeta(bkt, meta, "", "")
|
||||||
|
if err = n.objCache.Put(*meta); err != nil {
|
||||||
|
n.log.Warn("couldn't put obj to object cache",
|
||||||
|
zap.String("bucket name", objInfo.Bucket),
|
||||||
|
zap.Stringer("bucket cid", objInfo.CID()),
|
||||||
|
zap.String("object name", objInfo.Name),
|
||||||
|
zap.Stringer("object id", objInfo.ID()),
|
||||||
|
zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
return objInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectDelete puts tombstone object into neofs.
|
// objectDelete puts tombstone object into neofs.
|
||||||
func (n *layer) objectDelete(ctx context.Context, cid *cid.ID, oid *object.ID) error {
|
func (n *layer) objectDelete(ctx context.Context, cid *cid.ID, oid *object.ID) error {
|
||||||
address := object.NewAddress()
|
address := newAddress(cid, oid)
|
||||||
address.SetContainerID(cid)
|
|
||||||
address.SetObjectID(oid)
|
|
||||||
dop := new(client.DeleteObjectParams)
|
dop := new(client.DeleteObjectParams)
|
||||||
dop.WithAddress(address)
|
dop.WithAddress(address)
|
||||||
n.objCache.Delete(address)
|
n.objCache.Delete(address)
|
||||||
|
@ -390,35 +449,11 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) {
|
func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) {
|
||||||
ids, err := n.objectSearch(ctx, &findParams{cid: p.Bucket.CID})
|
versions, err := n.getAllObjectsVersions(ctx, p.Bucket, p.Prefix, p.Delimiter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
versions := make(map[string]*objectVersions, len(ids)/2)
|
|
||||||
|
|
||||||
for _, id := range ids {
|
|
||||||
meta, err := n.objectHead(ctx, p.Bucket.CID, id)
|
|
||||||
if err != nil {
|
|
||||||
n.log.Warn("could not fetch object meta", zap.Error(err))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if oi := objectInfoFromMeta(p.Bucket, meta, p.Prefix, p.Delimiter); oi != nil {
|
|
||||||
if isSystem(oi) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if objVersions, ok := versions[oi.Name]; ok {
|
|
||||||
objVersions.appendVersion(oi)
|
|
||||||
versions[oi.Name] = objVersions
|
|
||||||
} else {
|
|
||||||
objVersion := newObjectVersions(oi.Name)
|
|
||||||
objVersion.appendVersion(oi)
|
|
||||||
versions[oi.Name] = objVersion
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
objects := make([]*ObjectInfo, 0, len(versions))
|
objects := make([]*ObjectInfo, 0, len(versions))
|
||||||
for _, v := range versions {
|
for _, v := range versions {
|
||||||
lastVersion := v.getLast()
|
lastVersion := v.getLast()
|
||||||
|
@ -434,6 +469,38 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam
|
||||||
return objects, nil
|
return objects, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *cache.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) {
|
||||||
|
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
versions := make(map[string]*objectVersions, len(ids)/2)
|
||||||
|
for _, id := range ids {
|
||||||
|
meta, err := n.objectHead(ctx, bkt.CID, id)
|
||||||
|
if err != nil {
|
||||||
|
n.log.Warn("could not fetch object meta", zap.Error(err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if oi := objectInfoFromMeta(bkt, meta, prefix, delimiter); oi != nil {
|
||||||
|
if isSystem(oi) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if objVersions, ok := versions[oi.Name]; ok {
|
||||||
|
objVersions.appendVersion(oi)
|
||||||
|
versions[oi.Name] = objVersions
|
||||||
|
} else {
|
||||||
|
objVersion := newObjectVersions(oi.Name)
|
||||||
|
objVersion.appendVersion(oi)
|
||||||
|
versions[oi.Name] = objVersion
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return versions, nil
|
||||||
|
}
|
||||||
|
|
||||||
func getExistedVersions(versions *objectVersions) []string {
|
func getExistedVersions(versions *objectVersions) []string {
|
||||||
var res []string
|
var res []string
|
||||||
for _, add := range versions.addList {
|
for _, add := range versions.addList {
|
||||||
|
@ -498,7 +565,7 @@ func triageObjects(allObjects []*ObjectInfo) (prefixes []string, objects []*Obje
|
||||||
func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*ObjectInfo, error) {
|
func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*ObjectInfo, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
bkt *BucketInfo
|
bkt *cache.BucketInfo
|
||||||
cacheKey cacheOptions
|
cacheKey cacheOptions
|
||||||
allObjects []*ObjectInfo
|
allObjects []*ObjectInfo
|
||||||
)
|
)
|
||||||
|
@ -507,11 +574,11 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if cacheKey, err = createKey(ctx, bkt.CID, p.Prefix, p.Delimiter); err != nil {
|
if cacheKey, err = createKey(ctx, bkt.CID, listObjectsMethod, p.Prefix, p.Delimiter); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
allObjects = n.listObjCache.Get(cacheKey)
|
allObjects = n.listsCache.Get(cacheKey)
|
||||||
|
|
||||||
if allObjects == nil {
|
if allObjects == nil {
|
||||||
allObjects, err = n.listSortedObjectsFromNeoFS(ctx, allObjectParams{
|
allObjects, err = n.listSortedObjectsFromNeoFS(ctx, allObjectParams{
|
||||||
|
@ -524,13 +591,13 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
|
||||||
}
|
}
|
||||||
|
|
||||||
// putting to cache a copy of allObjects because allObjects can be modified further
|
// putting to cache a copy of allObjects because allObjects can be modified further
|
||||||
n.listObjCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...))
|
n.listsCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...))
|
||||||
}
|
}
|
||||||
|
|
||||||
return allObjects, nil
|
return allObjects, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *BucketInfo) bool {
|
func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *cache.BucketInfo) bool {
|
||||||
settings, err := n.getBucketSettings(ctx, bktInfo)
|
settings, err := n.getBucketSettings(ctx, bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.log.Warn("couldn't get versioning settings object", zap.Error(err))
|
n.log.Warn("couldn't get versioning settings object", zap.Error(err))
|
||||||
|
|
|
@ -30,6 +30,11 @@ type (
|
||||||
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
|
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
|
||||||
const DefaultObjectsListCacheLifetime = time.Second * 60
|
const DefaultObjectsListCacheLifetime = time.Second * 60
|
||||||
|
|
||||||
|
const (
|
||||||
|
listObjectsMethod = "listObjects"
|
||||||
|
listVersionsMethod = "listVersions"
|
||||||
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
listObjectsCache struct {
|
listObjectsCache struct {
|
||||||
cacheLifetime time.Duration
|
cacheLifetime time.Duration
|
||||||
|
@ -78,7 +83,7 @@ func (l *listObjectsCache) Put(key cacheOptions, objects []*ObjectInfo) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func createKey(ctx context.Context, cid *cid.ID, prefix, delimiter string) (cacheOptions, error) {
|
func createKey(ctx context.Context, cid *cid.ID, method, prefix, delimiter string) (cacheOptions, error) {
|
||||||
box, err := GetBoxData(ctx)
|
box, err := GetBoxData(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cacheOptions{}, err
|
return cacheOptions{}, err
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -60,7 +61,7 @@ type (
|
||||||
|
|
||||||
// ListObjectVersionsInfo stores info and list of objects' versions.
|
// ListObjectVersionsInfo stores info and list of objects' versions.
|
||||||
ListObjectVersionsInfo struct {
|
ListObjectVersionsInfo struct {
|
||||||
CommonPrefixes []*string
|
CommonPrefixes []string
|
||||||
IsTruncated bool
|
IsTruncated bool
|
||||||
KeyMarker string
|
KeyMarker string
|
||||||
NextKeyMarker string
|
NextKeyMarker string
|
||||||
|
@ -84,7 +85,11 @@ func userHeaders(attrs []*object.Attribute) map[string]string {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo {
|
func objInfoFromMeta(bkt *cache.BucketInfo, meta *object.Object) *ObjectInfo {
|
||||||
|
return objectInfoFromMeta(bkt, meta, "", "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectInfoFromMeta(bkt *cache.BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo {
|
||||||
var (
|
var (
|
||||||
isDir bool
|
isDir bool
|
||||||
size int64
|
size int64
|
||||||
|
@ -164,6 +169,12 @@ func (o *ObjectInfo) ID() *object.ID { return o.id }
|
||||||
// Version returns object version from ObjectInfo.
|
// Version returns object version from ObjectInfo.
|
||||||
func (o *ObjectInfo) Version() string { return o.id.String() }
|
func (o *ObjectInfo) Version() string { return o.id.String() }
|
||||||
|
|
||||||
|
// NiceName returns object name for cache.
|
||||||
|
func (o *ObjectInfo) NiceName() string { return o.Bucket + "/" + o.Name }
|
||||||
|
|
||||||
|
// Address returns object address.
|
||||||
|
func (o *ObjectInfo) Address() *object.Address { return newAddress(o.bucketID, o.id) }
|
||||||
|
|
||||||
// CID returns bucket ID from ObjectInfo.
|
// CID returns bucket ID from ObjectInfo.
|
||||||
func (o *ObjectInfo) CID() *cid.ID { return o.bucketID }
|
func (o *ObjectInfo) CID() *cid.ID { return o.bucketID }
|
||||||
|
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -19,7 +20,7 @@ var (
|
||||||
defaultTestContentType = http.DetectContentType(defaultTestPayload)
|
defaultTestContentType = http.DetectContentType(defaultTestPayload)
|
||||||
)
|
)
|
||||||
|
|
||||||
func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object {
|
func newTestObject(oid *object.ID, bkt *cache.BucketInfo, name string) *object.Object {
|
||||||
filename := object.NewAttribute()
|
filename := object.NewAttribute()
|
||||||
filename.SetKey(object.AttributeFileName)
|
filename.SetKey(object.AttributeFileName)
|
||||||
filename.SetValue(name)
|
filename.SetValue(name)
|
||||||
|
@ -43,7 +44,7 @@ func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object
|
||||||
return raw.Object()
|
return raw.Object()
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *ObjectInfo {
|
func newTestInfo(oid *object.ID, bkt *cache.BucketInfo, name string, isDir bool) *ObjectInfo {
|
||||||
info := &ObjectInfo{
|
info := &ObjectInfo{
|
||||||
id: oid,
|
id: oid,
|
||||||
Name: name,
|
Name: name,
|
||||||
|
@ -71,7 +72,7 @@ func Test_objectInfoFromMeta(t *testing.T) {
|
||||||
oid := object.NewID()
|
oid := object.NewID()
|
||||||
containerID := cid.New()
|
containerID := cid.New()
|
||||||
|
|
||||||
bkt := &BucketInfo{
|
bkt := &cache.BucketInfo{
|
||||||
Name: "test-container",
|
Name: "test-container",
|
||||||
CID: containerID,
|
CID: containerID,
|
||||||
Owner: uid,
|
Owner: uid,
|
||||||
|
|
|
@ -20,6 +20,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/session"
|
"github.com/nspcc-dev/neofs-api-go/pkg/session"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/token"
|
"github.com/nspcc-dev/neofs-api-go/pkg/token"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pkg/logger"
|
"github.com/nspcc-dev/neofs-sdk-go/pkg/logger"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pkg/pool"
|
"github.com/nspcc-dev/neofs-sdk-go/pkg/pool"
|
||||||
|
@ -61,10 +62,7 @@ func (t *testPool) PutObject(ctx context.Context, params *client.PutObjectParams
|
||||||
raw.SetPayload(all)
|
raw.SetPayload(all)
|
||||||
}
|
}
|
||||||
|
|
||||||
addr := object.NewAddress()
|
addr := newAddress(raw.ContainerID(), raw.ID())
|
||||||
addr.SetObjectID(raw.ID())
|
|
||||||
addr.SetContainerID(raw.ContainerID())
|
|
||||||
|
|
||||||
t.objects[addr.String()] = raw.Object()
|
t.objects[addr.String()] = raw.Object()
|
||||||
return raw.ID(), nil
|
return raw.ID(), nil
|
||||||
}
|
}
|
||||||
|
@ -329,8 +327,12 @@ func prepareContext(t *testing.T) *testContext {
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
return &testContext{
|
return &testContext{
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
layer: NewLayer(l, tp),
|
layer: NewLayer(l, tp, &CacheConfig{
|
||||||
|
Size: cache.DefaultObjectsCacheSize,
|
||||||
|
Lifetime: cache.DefaultObjectsCacheLifetime,
|
||||||
|
ListObjectsLifetime: DefaultObjectsListCacheLifetime},
|
||||||
|
),
|
||||||
bkt: bktName,
|
bkt: bktName,
|
||||||
bktID: bktID,
|
bktID: bktID,
|
||||||
obj: "obj1",
|
obj: "obj1",
|
||||||
|
|
Loading…
Reference in a new issue