From 19b917e3b578715a9da565c53a3babfab7b054af Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 10 Sep 2021 09:56:56 +0300 Subject: [PATCH] [#253] Caches refactoring Signed-off-by: Denis Kirillov --- api/auth/center.go | 3 +- .../cache/accessbox.go | 42 +++---- api/cache/buckets.go | 45 ++++---- api/cache/names.go | 39 +++---- api/cache/objects.go | 34 +++--- api/cache/objects_test.go | 14 ++- api/cache/objectslist.go | 49 ++++----- api/cache/objectslist_test.go | 32 ++++-- api/cache/system.go | 45 ++++---- api/{ => data}/info.go | 2 +- api/handler/acl.go | 3 +- api/handler/copy.go | 3 +- api/handler/get.go | 7 +- api/handler/get_test.go | 10 +- api/handler/head.go | 3 +- api/handler/object_list.go | 5 +- api/handler/put.go | 3 +- api/layer/container.go | 13 ++- api/layer/layer.go | 104 ++++++++++-------- api/layer/object.go | 41 +++---- api/layer/object_test.go | 12 +- api/layer/util.go | 11 +- api/layer/util_test.go | 12 +- api/layer/versioning.go | 26 ++--- api/layer/versioning_test.go | 69 ++++++------ authmate/authmate.go | 5 +- cmd/s3-gw/app.go | 33 +++--- cmd/s3-gw/app_settings.go | 6 + creds/tokens/credentials.go | 7 +- docs/configuration.md | 9 ++ 30 files changed, 365 insertions(+), 322 deletions(-) rename creds/tokens/accessbox_cache.go => api/cache/accessbox.go (50%) rename api/{ => data}/info.go (99%) diff --git a/api/auth/center.go b/api/auth/center.go index 613c803..c4c7b6a 100644 --- a/api/auth/center.go +++ b/api/auth/center.go @@ -18,6 +18,7 @@ import ( v4 "github.com/aws/aws-sdk-go/aws/signer/v4" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/tokens" @@ -81,7 +82,7 @@ func (p prs) Seek(_ int64, _ int) (int64, error) { var _ io.ReadSeeker = prs(0) // New creates an instance of AuthCenter. -func New(conns pool.Pool, key *keys.PrivateKey, config *tokens.CacheConfig) Center { +func New(conns pool.Pool, key *keys.PrivateKey, config *cache.Config) Center { return ¢er{ cli: tokens.New(conns, key, config), reg: ®expSubmatcher{re: authorizationFieldRegexp}, diff --git a/creds/tokens/accessbox_cache.go b/api/cache/accessbox.go similarity index 50% rename from creds/tokens/accessbox_cache.go rename to api/cache/accessbox.go index c7b2b27..b31b5ec 100644 --- a/creds/tokens/accessbox_cache.go +++ b/api/cache/accessbox.go @@ -1,4 +1,4 @@ -package tokens +package cache import ( "time" @@ -8,31 +8,33 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" ) -const ( - // DefaultCacheSize is a default maximum number of entries in cache. - DefaultCacheSize = 100 - // DefaultCacheLifetime is a default lifetime of entries in cache. - DefaultCacheLifetime = 10 * time.Minute +type ( + // AccessBoxCache stores access box by its address. + AccessBoxCache struct { + cache gcache.Cache + } + + // Config stores expiration params for cache. + Config struct { + Size int + Lifetime time.Duration + } ) -// CacheConfig stores expiration params for cache. -type CacheConfig struct { - Size int - Lifetime time.Duration -} +const ( + // DefaultAccessBoxCacheSize is a default maximum number of entries in cache. + DefaultAccessBoxCacheSize = 100 + // DefaultAccessBoxCacheLifetime is a default lifetime of entries in cache. + DefaultAccessBoxCacheLifetime = 10 * time.Minute +) -// DefaultCacheConfig return new default cache expiration values. -func DefaultCacheConfig() *CacheConfig { - return &CacheConfig{Size: DefaultCacheSize, Lifetime: DefaultCacheLifetime} -} - -// AccessBoxCache stores access box by its address. -type AccessBoxCache struct { - cache gcache.Cache +// DefaultAccessBoxConfig return new default cache expiration values. +func DefaultAccessBoxConfig() *Config { + return &Config{Size: DefaultAccessBoxCacheSize, Lifetime: DefaultAccessBoxCacheLifetime} } // NewAccessBoxCache creates an object of BucketCache. -func NewAccessBoxCache(config *CacheConfig) *AccessBoxCache { +func NewAccessBoxCache(config *Config) *AccessBoxCache { gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() return &AccessBoxCache{cache: gc} diff --git a/api/cache/buckets.go b/api/cache/buckets.go index 8576430..28197b9 100644 --- a/api/cache/buckets.go +++ b/api/cache/buckets.go @@ -4,39 +4,40 @@ import ( "time" "github.com/bluele/gcache" - "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" ) -type ( - // BucketCache provides interface for lru cache for objects. - BucketCache interface { - Get(key string) *api.BucketInfo - Put(bkt *api.BucketInfo) error - Delete(key string) bool - } +// BucketCache contains cache with objects and lifetime of cache entries. +type BucketCache struct { + cache gcache.Cache +} - // GetBucketCache contains cache with objects and lifetime of cache entries. - GetBucketCache struct { - cache gcache.Cache - lifetime time.Duration - } +const ( + // DefaultBucketCacheSize is a default maximum number of entries in cache. + DefaultBucketCacheSize = 1e3 + // DefaultBucketCacheLifetime is a default lifetime of entries in cache. + DefaultBucketCacheLifetime = time.Minute ) +// DefaultBucketConfig return new default cache expiration values. +func DefaultBucketConfig() *Config { + return &Config{Size: DefaultBucketCacheSize, Lifetime: DefaultBucketCacheLifetime} +} + // NewBucketCache creates an object of BucketCache. -func NewBucketCache(cacheSize int, lifetime time.Duration) *GetBucketCache { - gc := gcache.New(cacheSize).LRU().Build() - - return &GetBucketCache{cache: gc, lifetime: lifetime} +func NewBucketCache(config *Config) *BucketCache { + gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + return &BucketCache{cache: gc} } // Get returns cached object. -func (o *GetBucketCache) Get(key string) *api.BucketInfo { +func (o *BucketCache) Get(key string) *data.BucketInfo { entry, err := o.cache.Get(key) if err != nil { return nil } - result, ok := entry.(*api.BucketInfo) + result, ok := entry.(*data.BucketInfo) if !ok { return nil } @@ -45,11 +46,11 @@ func (o *GetBucketCache) Get(key string) *api.BucketInfo { } // Put puts an object to cache. -func (o *GetBucketCache) Put(bkt *api.BucketInfo) error { - return o.cache.SetWithExpire(bkt.Name, bkt, o.lifetime) +func (o *BucketCache) Put(bkt *data.BucketInfo) error { + return o.cache.Set(bkt.Name, bkt) } // Delete deletes an object from cache. -func (o *GetBucketCache) Delete(key string) bool { +func (o *BucketCache) Delete(key string) bool { return o.cache.Remove(key) } diff --git a/api/cache/names.go b/api/cache/names.go index afb4b57..b205519 100644 --- a/api/cache/names.go +++ b/api/cache/names.go @@ -7,32 +7,33 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" ) -// ObjectsNameCache provides interface for lru cache for objects. +// ObjectsNameCache provides for lru cache for objects. // This cache contains mapping nice name to object addresses. // Key is bucketName+objectName. -type ObjectsNameCache interface { - Get(key string) *object.Address - Put(key string, address *object.Address) error - Delete(key string) bool +type ObjectsNameCache struct { + cache gcache.Cache } -type ( - // NameCache contains cache with objects and lifetime of cache entries. - NameCache struct { - cache gcache.Cache - lifetime time.Duration - } +const ( + // DefaultObjectsNameCacheSize is a default maximum number of entries in cache. + DefaultObjectsNameCacheSize = 1e4 + // DefaultObjectsNameCacheLifetime is a default lifetime of entries in cache. + DefaultObjectsNameCacheLifetime = time.Minute ) -// NewObjectsNameCache creates an object of ObjectsNameCache. -func NewObjectsNameCache(cacheSize int, lifetime time.Duration) *NameCache { - gc := gcache.New(cacheSize).LRU().Build() +// DefaultObjectsNameConfig return new default cache expiration values. +func DefaultObjectsNameConfig() *Config { + return &Config{Size: DefaultObjectsNameCacheSize, Lifetime: DefaultObjectsNameCacheLifetime} +} - return &NameCache{cache: gc, lifetime: lifetime} +// NewObjectsNameCache creates an object of ObjectsNameCache. +func NewObjectsNameCache(config *Config) *ObjectsNameCache { + gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + return &ObjectsNameCache{cache: gc} } // Get returns cached object. -func (o *NameCache) Get(key string) *object.Address { +func (o *ObjectsNameCache) Get(key string) *object.Address { entry, err := o.cache.Get(key) if err != nil { return nil @@ -47,11 +48,11 @@ func (o *NameCache) Get(key string) *object.Address { } // Put puts an object to cache. -func (o *NameCache) Put(key string, address *object.Address) error { - return o.cache.SetWithExpire(key, address, o.lifetime) +func (o *ObjectsNameCache) Put(key string, address *object.Address) error { + return o.cache.Set(key, address) } // Delete deletes an object from cache. -func (o *NameCache) Delete(key string) bool { +func (o *ObjectsNameCache) Delete(key string) bool { return o.cache.Remove(key) } diff --git a/api/cache/objects.go b/api/cache/objects.go index 708fb69..f7e70d3 100644 --- a/api/cache/objects.go +++ b/api/cache/objects.go @@ -7,11 +7,9 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" ) -// ObjectsCache provides interface for lru cache for objects. -type ObjectsCache interface { - Get(address *object.Address) *object.Object - Put(obj object.Object) error - Delete(address *object.Address) bool +// ObjectsCache provides lru cache for objects. +type ObjectsCache struct { + cache gcache.Cache } const ( @@ -21,23 +19,19 @@ const ( DefaultObjectsCacheSize = 1e6 ) -type ( - // ObjectHeadersCache contains cache with objects and lifetime of cache entries. - ObjectHeadersCache struct { - cache gcache.Cache - lifetime time.Duration - } -) +// DefaultObjectsConfig return new default cache expiration values. +func DefaultObjectsConfig() *Config { + return &Config{Size: DefaultObjectsCacheSize, Lifetime: DefaultObjectsCacheLifetime} +} // New creates an object of ObjectHeadersCache. -func New(cacheSize int, lifetime time.Duration) *ObjectHeadersCache { - gc := gcache.New(cacheSize).LRU().Build() - - return &ObjectHeadersCache{cache: gc, lifetime: lifetime} +func New(config *Config) *ObjectsCache { + gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + return &ObjectsCache{cache: gc} } // Get returns cached object. -func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object { +func (o *ObjectsCache) Get(address *object.Address) *object.Object { entry, err := o.cache.Get(address.String()) if err != nil { return nil @@ -52,11 +46,11 @@ func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object { } // Put puts an object to cache. -func (o *ObjectHeadersCache) Put(obj object.Object) error { - return o.cache.SetWithExpire(obj.ContainerID().String()+"/"+obj.ID().String(), obj, o.lifetime) +func (o *ObjectsCache) Put(obj object.Object) error { + return o.cache.Set(obj.ContainerID().String()+"/"+obj.ID().String(), obj) } // Delete deletes an object from cache. -func (o *ObjectHeadersCache) Delete(address *object.Address) bool { +func (o *ObjectsCache) Delete(address *object.Address) bool { return o.cache.Remove(address.String()) } diff --git a/api/cache/objects_test.go b/api/cache/objects_test.go index 40dfd87..2a22235 100644 --- a/api/cache/objects_test.go +++ b/api/cache/objects_test.go @@ -9,10 +9,12 @@ import ( "github.com/stretchr/testify/require" ) -const ( - cachesize = 10 - lifetime = time.Second * 5 -) +func getTestConfig() *Config { + return &Config{ + Size: 10, + Lifetime: 5 * time.Second, + } +} func TestCache(t *testing.T) { obj := objecttest.Object() @@ -21,7 +23,7 @@ func TestCache(t *testing.T) { address.SetObjectID(obj.ID()) t.Run("check get", func(t *testing.T) { - cache := New(cachesize, lifetime) + cache := New(getTestConfig()) err := cache.Put(*obj) require.NoError(t, err) @@ -30,7 +32,7 @@ func TestCache(t *testing.T) { }) t.Run("check delete", func(t *testing.T) { - cache := New(cachesize, lifetime) + cache := New(getTestConfig()) err := cache.Put(*obj) require.NoError(t, err) diff --git a/api/cache/objectslist.go b/api/cache/objectslist.go index 97d5853..f3b58e0 100644 --- a/api/cache/objectslist.go +++ b/api/cache/objectslist.go @@ -24,11 +24,15 @@ import ( */ type ( - // ObjectsListCache provides interface for cache of ListObjectsV2 in a layer struct. - ObjectsListCache interface { - Get(key ObjectsListKey) []*object.ID - Put(key ObjectsListKey, oids []*object.ID) error - CleanCacheEntriesContainingObject(objectName string, cid *cid.ID) + // ObjectsListCache contains cache for ListObjects and ListObjectVersions. + ObjectsListCache struct { + cache gcache.Cache + } + + // ObjectsListKey is a key to find a ObjectsListCache's entry. + ObjectsListKey struct { + cid string + prefix string } ) @@ -39,32 +43,19 @@ const ( DefaultObjectsListCacheSize = 1e5 ) -type ( - // ListObjectsCache contains cache for ListObjects and ListObjectVersions. - ListObjectsCache struct { - lifetime time.Duration - cache gcache.Cache - } - - // ObjectsListKey is a key to find a ObjectsListCache's entry. - ObjectsListKey struct { - cid string - prefix string - } -) +// DefaultObjectsListConfig return new default cache expiration values. +func DefaultObjectsListConfig() *Config { + return &Config{Size: DefaultObjectsListCacheSize, Lifetime: DefaultObjectsListCacheLifetime} +} // NewObjectsListCache is a constructor which creates an object of ListObjectsCache with given lifetime of entries. -func NewObjectsListCache(cacheSize int, lifetime time.Duration) *ListObjectsCache { - gc := gcache.New(cacheSize).LRU().Build() - - return &ListObjectsCache{ - cache: gc, - lifetime: lifetime, - } +func NewObjectsListCache(config *Config) *ObjectsListCache { + gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + return &ObjectsListCache{cache: gc} } // Get return list of ObjectInfo. -func (l *ListObjectsCache) Get(key ObjectsListKey) []*object.ID { +func (l *ObjectsListCache) Get(key ObjectsListKey) []*object.ID { entry, err := l.cache.Get(key) if err != nil { return nil @@ -79,16 +70,16 @@ func (l *ListObjectsCache) Get(key ObjectsListKey) []*object.ID { } // Put puts a list of objects to cache. -func (l *ListObjectsCache) Put(key ObjectsListKey, oids []*object.ID) error { +func (l *ObjectsListCache) Put(key ObjectsListKey, oids []*object.ID) error { if len(oids) == 0 { return fmt.Errorf("list is empty, cid: %s, prefix: %s", key.cid, key.prefix) } - return l.cache.SetWithExpire(key, oids, l.lifetime) + return l.cache.Set(key, oids) } // CleanCacheEntriesContainingObject deletes entries containing specified object. -func (l *ListObjectsCache) CleanCacheEntriesContainingObject(objectName string, cid *cid.ID) { +func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string, cid *cid.ID) { cidStr := cid.String() keys := l.cache.Keys(true) for _, key := range keys { diff --git a/api/cache/objectslist_test.go b/api/cache/objectslist_test.go index d305ec4..8569dfe 100644 --- a/api/cache/objectslist_test.go +++ b/api/cache/objectslist_test.go @@ -14,6 +14,13 @@ import ( const testingCacheLifetime = 5 * time.Second const testingCacheSize = 10 +func getTestObjectsListConfig() *Config { + return &Config{ + Size: testingCacheSize, + Lifetime: testingCacheLifetime, + } +} + func randID(t *testing.T) *object.ID { id := object.NewID() id.SetSHA256(randSHA256Checksum(t)) @@ -42,7 +49,8 @@ func TestObjectsListCache(t *testing.T) { t.Run("lifetime", func(t *testing.T) { var ( - cache = NewObjectsListCache(testingCacheSize, testingCacheLifetime) + config = getTestObjectsListConfig() + cache = NewObjectsListCache(config) cacheKey = ObjectsListKey{cid: userKey} ) @@ -53,13 +61,13 @@ func TestObjectsListCache(t *testing.T) { return cache.Get(cacheKey) == nil } - require.Never(t, condition, cache.lifetime, time.Second) + require.Never(t, condition, config.Lifetime, time.Second) require.Eventually(t, condition, time.Second, 10*time.Millisecond) }) t.Run("get cache with empty prefix", func(t *testing.T) { var ( - cache = NewObjectsListCache(testingCacheSize, testingCacheLifetime) + cache = NewObjectsListCache(getTestObjectsListConfig()) cacheKey = ObjectsListKey{cid: userKey} ) err := cache.Put(cacheKey, ids) @@ -79,7 +87,7 @@ func TestObjectsListCache(t *testing.T) { prefix: "dir", } - cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime) + cache := NewObjectsListCache(getTestObjectsListConfig()) err := cache.Put(cacheKey, ids) require.NoError(t, err) @@ -104,7 +112,7 @@ func TestObjectsListCache(t *testing.T) { } ) - cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime) + cache := NewObjectsListCache(getTestObjectsListConfig()) err := cache.Put(cacheKey, ids) require.NoError(t, err) @@ -122,7 +130,7 @@ func TestObjectsListCache(t *testing.T) { } ) - cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime) + cache := NewObjectsListCache(getTestObjectsListConfig()) err := cache.Put(cacheKey, ids) require.NoError(t, err) @@ -143,7 +151,9 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) { } t.Run("put object to the root of the bucket", func(t *testing.T) { - cache := NewObjectsListCache(testingCacheSize, time.Minute) + config := getTestObjectsListConfig() + config.Lifetime = time.Minute + cache := NewObjectsListCache(config) for _, k := range keys { err := cache.Put(k, oids) require.NoError(t, err) @@ -160,7 +170,9 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) { }) t.Run("put object to dir/", func(t *testing.T) { - cache := NewObjectsListCache(testingCacheSize, time.Minute) + config := getTestObjectsListConfig() + config.Lifetime = time.Minute + cache := NewObjectsListCache(config) for _, k := range keys { err := cache.Put(k, oids) require.NoError(t, err) @@ -177,7 +189,9 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) { }) t.Run("put object to dir/lol/", func(t *testing.T) { - cache := NewObjectsListCache(testingCacheSize, time.Minute) + config := getTestObjectsListConfig() + config.Lifetime = time.Minute + cache := NewObjectsListCache(config) for _, k := range keys { err := cache.Put(k, oids) require.NoError(t, err) diff --git a/api/cache/system.go b/api/cache/system.go index 7ceec01..3c6583c 100644 --- a/api/cache/system.go +++ b/api/cache/system.go @@ -7,32 +7,33 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" ) -type ( - // SystemCache provides interface for lru cache for objects. - // This cache contains "system" objects (bucket versioning settings, tagging object etc.). - // Key is bucketName+systemFileName. - SystemCache interface { - Get(key string) *object.Object - Put(key string, obj *object.Object) error - Delete(key string) bool - } +// SystemCache provides lru cache for objects. +// This cache contains "system" objects (bucket versioning settings, tagging object etc.). +// Key is bucketName+systemFileName. +type SystemCache struct { + cache gcache.Cache +} - // SysCache contains cache with objects and lifetime of cache entries. - SysCache struct { - cache gcache.Cache - lifetime time.Duration - } +const ( + // DefaultSystemCacheSize is a default maximum number of entries in cache. + DefaultSystemCacheSize = 1e4 + // DefaultSystemCacheLifetime is a default lifetime of entries in cache. + DefaultSystemCacheLifetime = 5 * time.Minute ) -// NewSystemCache creates an object of SystemCache. -func NewSystemCache(cacheSize int, lifetime time.Duration) *SysCache { - gc := gcache.New(cacheSize).LRU().Build() +// DefaultSystemConfig return new default cache expiration values. +func DefaultSystemConfig() *Config { + return &Config{Size: DefaultSystemCacheSize, Lifetime: DefaultSystemCacheLifetime} +} - return &SysCache{cache: gc, lifetime: lifetime} +// NewSystemCache creates an object of SystemCache. +func NewSystemCache(config *Config) *SystemCache { + gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + return &SystemCache{cache: gc} } // Get returns cached object. -func (o *SysCache) Get(key string) *object.Object { +func (o *SystemCache) Get(key string) *object.Object { entry, err := o.cache.Get(key) if err != nil { return nil @@ -47,11 +48,11 @@ func (o *SysCache) Get(key string) *object.Object { } // Put puts an object to cache. -func (o *SysCache) Put(key string, obj *object.Object) error { - return o.cache.SetWithExpire(key, obj, o.lifetime) +func (o *SystemCache) Put(key string, obj *object.Object) error { + return o.cache.Set(key, obj) } // Delete deletes an object from cache. -func (o *SysCache) Delete(key string) bool { +func (o *SystemCache) Delete(key string) bool { return o.cache.Remove(key) } diff --git a/api/info.go b/api/data/info.go similarity index 99% rename from api/info.go rename to api/data/info.go index 2dd57de..dc3b5a4 100644 --- a/api/info.go +++ b/api/data/info.go @@ -1,4 +1,4 @@ -package api +package data import ( "time" diff --git a/api/handler/acl.go b/api/handler/acl.go index 1b3285a..16b24cb 100644 --- a/api/handler/acl.go +++ b/api/handler/acl.go @@ -14,6 +14,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/v2/acl" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" ) @@ -299,7 +300,7 @@ func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) } } -func checkOwner(info *api.BucketInfo, owner string) error { +func checkOwner(info *data.BucketInfo, owner string) error { if owner == "" { return nil } diff --git a/api/handler/copy.go b/api/handler/copy.go index 8cf1f7e..897f585 100644 --- a/api/handler/copy.go +++ b/api/handler/copy.go @@ -7,6 +7,7 @@ import ( "time" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "go.uber.org/zap" @@ -32,7 +33,7 @@ func path2BucketObject(path string) (bucket, prefix string) { func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { var ( err error - info *api.ObjectInfo + info *data.ObjectInfo metadata map[string]string reqInfo = api.GetReqInfo(r.Context()) diff --git a/api/handler/get.go b/api/handler/get.go index ada8571..bbd17c9 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -8,6 +8,7 @@ import ( "time" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" ) @@ -65,7 +66,7 @@ func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, return &layer.RangeParams{Start: start, End: end}, nil } -func writeHeaders(h http.Header, info *api.ObjectInfo, tagSetLength int) { +func writeHeaders(h http.Header, info *data.ObjectInfo, tagSetLength int) { if len(info.ContentType) > 0 { h.Set(api.ContentType, info.ContentType) } @@ -83,7 +84,7 @@ func writeHeaders(h http.Header, info *api.ObjectInfo, tagSetLength int) { func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { var ( err error - info *api.ObjectInfo + info *data.ObjectInfo params *layer.RangeParams reqInfo = api.GetReqInfo(r.Context()) @@ -143,7 +144,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { } } -func checkPreconditions(info *api.ObjectInfo, args *conditionalArgs) error { +func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error { if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum { return errors.GetAPIError(errors.ErrPreconditionFailed) } diff --git a/api/handler/get_test.go b/api/handler/get_test.go index d27c3a4..ba415f3 100644 --- a/api/handler/get_test.go +++ b/api/handler/get_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/stretchr/testify/require" @@ -46,8 +46,8 @@ func TestFetchRangeHeader(t *testing.T) { } } -func newInfo(etag string, created time.Time) *api.ObjectInfo { - return &api.ObjectInfo{ +func newInfo(etag string, created time.Time) *data.ObjectInfo { + return &data.ObjectInfo{ HashSum: etag, Created: created, } @@ -61,13 +61,13 @@ func TestPreconditions(t *testing.T) { for _, tc := range []struct { name string - info *api.ObjectInfo + info *data.ObjectInfo args *conditionalArgs expected error }{ { name: "no conditions", - info: new(api.ObjectInfo), + info: new(data.ObjectInfo), args: new(conditionalArgs), expected: nil, }, diff --git a/api/handler/head.go b/api/handler/head.go index 36d8326..2f9c662 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "go.uber.org/zap" @@ -27,7 +28,7 @@ func getRangeToDetectContentType(maxSize int64) *layer.RangeParams { func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { var ( err error - info *api.ObjectInfo + info *data.ObjectInfo reqInfo = api.GetReqInfo(r.Context()) ) diff --git a/api/handler/object_list.go b/api/handler/object_list.go index db876bb..e978fb7 100644 --- a/api/handler/object_list.go +++ b/api/handler/object_list.go @@ -8,6 +8,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" ) @@ -183,11 +184,11 @@ func fillPrefixes(src []string, encode string) []CommonPrefix { return dst } -func fillContentsWithOwner(src []*api.ObjectInfo, encode string) []Object { +func fillContentsWithOwner(src []*data.ObjectInfo, encode string) []Object { return fillContents(src, encode, true) } -func fillContents(src []*api.ObjectInfo, encode string, fetchOwner bool) []Object { +func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Object { var dst []Object for _, obj := range src { res := Object{ diff --git a/api/handler/put.go b/api/handler/put.go index 191b46b..1794fcd 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -17,6 +17,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/auth" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "go.uber.org/zap" @@ -419,7 +420,7 @@ func containsACLHeaders(r *http.Request) bool { r.Header.Get(api.AmzGrantFullControl) != "" || r.Header.Get(api.AmzGrantWrite) != "" } -func (h *handler) getNewEAclTable(r *http.Request, objInfo *api.ObjectInfo) (*eacl.Table, error) { +func (h *handler) getNewEAclTable(r *http.Request, objInfo *data.ObjectInfo) (*eacl.Table, error) { var newEaclTable *eacl.Table objectACL, err := parseACLHeaders(r) if err != nil { diff --git a/api/layer/container.go b/api/layer/container.go index f2fad63..0be4739 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -12,6 +12,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/container" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-sdk-go/pool" "go.uber.org/zap" @@ -20,19 +21,19 @@ import ( type ( // BucketACL extends BucketInfo by eacl.Table. BucketACL struct { - Info *api.BucketInfo + Info *data.BucketInfo EACL *eacl.Table } ) -func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*api.BucketInfo, error) { +func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*data.BucketInfo, error) { var ( err error res *container.Container rid = api.GetRequestID(ctx) bearerOpt = n.BearerOpt(ctx) - info = &api.BucketInfo{ + info = &data.BucketInfo{ CID: cid, Name: cid.String(), } @@ -83,7 +84,7 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*api.BucketInfo return info, nil } -func (n *layer) containerList(ctx context.Context) ([]*api.BucketInfo, error) { +func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) { var ( err error own = n.Owner(ctx) @@ -99,7 +100,7 @@ func (n *layer) containerList(ctx context.Context) ([]*api.BucketInfo, error) { return nil, err } - list := make([]*api.BucketInfo, 0, len(res)) + list := make([]*data.BucketInfo, 0, len(res)) for _, cid := range res { info, err := n.containerInfo(ctx, cid) if err != nil { @@ -117,7 +118,7 @@ func (n *layer) containerList(ctx context.Context) ([]*api.BucketInfo, error) { func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*cid.ID, error) { var err error - bktInfo := &api.BucketInfo{ + bktInfo := &data.BucketInfo{ Name: p.Name, Owner: n.Owner(ctx), Created: time.Now(), diff --git a/api/layer/layer.go b/api/layer/layer.go index b9cf5ce..b04ae1b 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -19,6 +19,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/cache" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-sdk-go/pool" @@ -29,19 +30,20 @@ type ( layer struct { pool pool.Pool log *zap.Logger - listsCache cache.ObjectsListCache - objCache cache.ObjectsCache - namesCache cache.ObjectsNameCache - bucketCache cache.BucketCache - systemCache cache.SystemCache + listsCache *cache.ObjectsListCache + objCache *cache.ObjectsCache + namesCache *cache.ObjectsNameCache + bucketCache *cache.BucketCache + systemCache *cache.SystemCache } - // CacheConfig contains params for caches. - CacheConfig struct { - Lifetime time.Duration - Size int - ListObjectsLifetime time.Duration - ListObjectsSize int + // CachesConfig contains params for caches. + CachesConfig struct { + Objects *cache.Config + ObjectsList *cache.Config + Names *cache.Config + Buckets *cache.Config + System *cache.Config } // Params stores basic API parameters. @@ -55,7 +57,7 @@ type ( // GetObjectParams stores object get request parameters. GetObjectParams struct { Range *RangeParams - ObjectInfo *api.ObjectInfo + ObjectInfo *data.ObjectInfo Offset int64 Length int64 Writer io.Writer @@ -97,7 +99,7 @@ type ( // CopyObjectParams stores object copy request parameters. CopyObjectParams struct { - SrcObject *api.ObjectInfo + SrcObject *data.ObjectInfo DstBucket string DstObject string SrcSize int64 @@ -141,7 +143,7 @@ type ( // PutTaggingParams stores tag set params. PutTaggingParams struct { - ObjectInfo *api.ObjectInfo + ObjectInfo *data.ObjectInfo TagSet map[string]string } @@ -154,33 +156,33 @@ type ( Client interface { NeoFS - PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*api.ObjectInfo, error) + PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*data.ObjectInfo, error) GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error) - ListBuckets(ctx context.Context) ([]*api.BucketInfo, error) - GetBucketInfo(ctx context.Context, name string) (*api.BucketInfo, error) + ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) + GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error) GetBucketACL(ctx context.Context, name string) (*BucketACL, error) PutBucketACL(ctx context.Context, p *PutBucketACLParams) error CreateBucket(ctx context.Context, p *CreateBucketParams) (*cid.ID, error) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error GetObject(ctx context.Context, p *GetObjectParams) error - GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*api.ObjectInfo, error) - GetObjectTagging(ctx context.Context, p *api.ObjectInfo) (map[string]string, error) + GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) + GetObjectTagging(ctx context.Context, p *data.ObjectInfo) (map[string]string, error) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) - PutObject(ctx context.Context, p *PutObjectParams) (*api.ObjectInfo, error) + PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error PutBucketTagging(ctx context.Context, bucket string, tagSet map[string]string) error - CopyObject(ctx context.Context, p *CopyObjectParams) (*api.ObjectInfo, error) + CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ObjectInfo, error) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) ([]*VersionedObject, error) - DeleteObjectTagging(ctx context.Context, p *api.ObjectInfo) error + DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) error DeleteBucketTagging(ctx context.Context, bucket string) error } ) @@ -194,18 +196,28 @@ func (t *VersionedObject) String() string { return t.Name + ":" + t.VersionID } +// DefaultCachesConfigs returns filled configs. +func DefaultCachesConfigs() *CachesConfig { + return &CachesConfig{ + Objects: cache.DefaultObjectsConfig(), + ObjectsList: cache.DefaultObjectsListConfig(), + Names: cache.DefaultObjectsNameConfig(), + Buckets: cache.DefaultBucketConfig(), + System: cache.DefaultSystemConfig(), + } +} + // NewLayer creates instance of layer. It checks credentials // and establishes gRPC connection with node. -func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client { +func NewLayer(log *zap.Logger, conns pool.Pool, config *CachesConfig) Client { return &layer{ - pool: conns, - log: log, - listsCache: cache.NewObjectsListCache(config.ListObjectsSize, config.ListObjectsLifetime), - objCache: cache.New(config.Size, config.Lifetime), - //todo reconsider cache params - namesCache: cache.NewObjectsNameCache(1000, time.Minute), - bucketCache: cache.NewBucketCache(150, time.Minute), - systemCache: cache.NewSystemCache(1000, 5*time.Minute), + pool: conns, + log: log, + listsCache: cache.NewObjectsListCache(config.ObjectsList), + objCache: cache.New(config.Objects), + namesCache: cache.NewObjectsNameCache(config.Names), + bucketCache: cache.NewBucketCache(config.Buckets), + systemCache: cache.NewSystemCache(config.System), } } @@ -243,7 +255,7 @@ func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Objec } // GetBucketInfo returns bucket info by name. -func (n *layer) GetBucketInfo(ctx context.Context, name string) (*api.BucketInfo, error) { +func (n *layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error) { name, err := url.QueryUnescape(name) if err != nil { return nil, err @@ -301,7 +313,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err // ListBuckets returns all user containers. Name of the bucket is a container // id. Timestamp is omitted since it is not saved in neofs container. -func (n *layer) ListBuckets(ctx context.Context) ([]*api.BucketInfo, error) { +func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) { return n.containerList(ctx) } @@ -337,7 +349,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { } // GetObjectInfo returns meta information about the object. -func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*api.ObjectInfo, error) { +func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) { bkt, err := n.GetBucketInfo(ctx, p.Bucket) if err != nil { n.log.Error("could not fetch bucket info", zap.Error(err)) @@ -352,7 +364,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*api.Ob } // PutObject into storage. -func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*api.ObjectInfo, error) { +func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) { bkt, err := n.GetBucketInfo(ctx, p.Bucket) if err != nil { return nil, err @@ -362,8 +374,8 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*api.ObjectI } // GetObjectTagging from storage. -func (n *layer) GetObjectTagging(ctx context.Context, oi *api.ObjectInfo) (map[string]string, error) { - bktInfo := &api.BucketInfo{ +func (n *layer) GetObjectTagging(ctx context.Context, oi *data.ObjectInfo) (map[string]string, error) { + bktInfo := &data.BucketInfo{ Name: oi.Bucket, CID: oi.CID, Owner: oi.Owner, @@ -392,7 +404,7 @@ func (n *layer) GetBucketTagging(ctx context.Context, bucketName string) (map[st return formTagSet(objInfo), nil } -func formTagSet(objInfo *api.ObjectInfo) map[string]string { +func formTagSet(objInfo *data.ObjectInfo) map[string]string { var tagSet map[string]string if objInfo != nil { tagSet = make(map[string]string, len(objInfo.Headers)) @@ -410,7 +422,7 @@ func formTagSet(objInfo *api.ObjectInfo) map[string]string { // PutObjectTagging into storage. func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error { - bktInfo := &api.BucketInfo{ + bktInfo := &data.BucketInfo{ Name: p.ObjectInfo.Bucket, CID: p.ObjectInfo.CID, Owner: p.ObjectInfo.Owner, @@ -438,7 +450,7 @@ func (n *layer) PutBucketTagging(ctx context.Context, bucketName string, tagSet } // DeleteObjectTagging from storage. -func (n *layer) DeleteObjectTagging(ctx context.Context, p *api.ObjectInfo) error { +func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) error { bktInfo, err := n.GetBucketInfo(ctx, p.Bucket) if err != nil { return err @@ -446,7 +458,7 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *api.ObjectInfo) erro return n.deleteSystemObject(ctx, bktInfo, p.TagsObject()) } -func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *api.BucketInfo, name string) error { +func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error { var oid *object.ID if meta := n.systemCache.Get(bktInfo.SystemObjectKey(name)); meta != nil { oid = meta.ID() @@ -475,7 +487,7 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, bucketName string) erro return n.deleteSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName)) } -func (n *layer) putSystemObject(ctx context.Context, bktInfo *api.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.Object, error) { +func (n *layer) putSystemObject(ctx context.Context, bktInfo *data.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.Object, error) { var ( err error oldOID *object.ID @@ -542,7 +554,7 @@ func (n *layer) putSystemObject(ctx context.Context, bktInfo *api.BucketInfo, ob return meta, nil } -func (n *layer) getSystemObject(ctx context.Context, bkt *api.BucketInfo, objName string) (*api.ObjectInfo, error) { +func (n *layer) getSystemObject(ctx context.Context, bkt *data.BucketInfo, objName string) (*data.ObjectInfo, error) { if meta := n.systemCache.Get(bkt.SystemObjectKey(objName)); meta != nil { return objInfoFromMeta(bkt, meta), nil } @@ -564,7 +576,7 @@ func (n *layer) getSystemObject(ctx context.Context, bkt *api.BucketInfo, objNam } // CopyObject from one bucket into another bucket. -func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*api.ObjectInfo, error) { +func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ObjectInfo, error) { pr, pw := io.Pipe() go func() { @@ -588,7 +600,7 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*api.Objec } // DeleteObject removes all objects with passed nice name. -func (n *layer) deleteObject(ctx context.Context, bkt *api.BucketInfo, obj *VersionedObject) *VersionedObject { +func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) *VersionedObject { var ( err error ids []*object.ID @@ -642,7 +654,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *api.BucketInfo, obj *Vers obj.Error = err return obj } - if err = n.DeleteObjectTagging(ctx, &api.ObjectInfo{ID: id, Bucket: bkt.Name, Name: obj.Name}); err != nil { + if err = n.DeleteObjectTagging(ctx, &data.ObjectInfo{ID: id, Bucket: bkt.Name, Name: obj.Name}); err != nil { obj.Error = err return obj } diff --git a/api/layer/object.go b/api/layer/object.go index 161ea51..925c192 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -16,6 +16,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/cache" + "github.com/nspcc-dev/neofs-s3-gw/api/data" apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors" "go.uber.org/zap" ) @@ -62,7 +63,7 @@ type ( } allObjectParams struct { - Bucket *api.BucketInfo + Bucket *data.BucketInfo Delimiter string Prefix string } @@ -134,7 +135,7 @@ func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) { } // objectPut into NeoFS, took payload from io.Reader. -func (n *layer) objectPut(ctx context.Context, bkt *api.BucketInfo, p *PutObjectParams) (*api.ObjectInfo, error) { +func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjectParams) (*data.ObjectInfo, error) { own := n.Owner(ctx) obj, err := url.QueryUnescape(p.Object) if err != nil { @@ -198,7 +199,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *api.BucketInfo, p *PutObject } } - return &api.ObjectInfo{ + return &data.ObjectInfo{ ID: oid, CID: bkt.CID, @@ -287,7 +288,7 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio return idsToDeleteArr } -func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *api.BucketInfo, objectName string) (*api.ObjectInfo, error) { +func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ObjectInfo, error) { if address := n.namesCache.Get(bkt.Name + "/" + objectName); address != nil { if headInfo := n.objCache.Get(address); headInfo != nil { return objInfoFromMeta(bkt, headInfo), nil @@ -313,7 +314,7 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *api.Bucket return lastVersion, nil } -func (n *layer) headVersions(ctx context.Context, bkt *api.BucketInfo, objectName string) (*objectVersions, error) { +func (n *layer) headVersions(ctx context.Context, bkt *data.BucketInfo, objectName string) (*objectVersions, error) { ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName}) if err != nil { return nil, err @@ -351,7 +352,7 @@ func (n *layer) headVersions(ctx context.Context, bkt *api.BucketInfo, objectNam return versions, nil } -func (n *layer) headVersion(ctx context.Context, bkt *api.BucketInfo, versionID string) (*api.ObjectInfo, error) { +func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, versionID string) (*data.ObjectInfo, error) { oid := object.NewID() if err := oid.Parse(versionID); err != nil { return nil, err @@ -396,7 +397,7 @@ func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*Lis var ( err error result ListObjectsInfoV1 - allObjects []*api.ObjectInfo + allObjects []*data.ObjectInfo ) if p.MaxKeys == 0 { @@ -431,7 +432,7 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis var ( err error result ListObjectsInfoV2 - allObjects []*api.ObjectInfo + allObjects []*data.ObjectInfo ) if p.MaxKeys == 0 { @@ -465,13 +466,13 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis return &result, nil } -func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*api.ObjectInfo, error) { +func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*data.ObjectInfo, error) { versions, err := n.getAllObjectsVersions(ctx, p.Bucket, p.Prefix, p.Delimiter) if err != nil { return nil, err } - objects := make([]*api.ObjectInfo, 0, len(versions)) + objects := make([]*data.ObjectInfo, 0, len(versions)) for _, v := range versions { lastVersion := v.getLast() if lastVersion != nil { @@ -486,7 +487,7 @@ func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*ap return objects, nil } -func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *api.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) { +func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) { var err error cacheKey := cache.CreateObjectsListCacheKey(bkt.CID, prefix) @@ -534,12 +535,12 @@ func splitVersions(header string) []string { return strings.Split(header, ",") } -func isSystem(obj *api.ObjectInfo) bool { +func isSystem(obj *data.ObjectInfo) bool { return len(obj.Headers[objectSystemAttributeName]) > 0 || len(obj.Headers[attrVersionsIgnore]) > 0 } -func trimAfterObjectName(startAfter string, objects []*api.ObjectInfo) []*api.ObjectInfo { +func trimAfterObjectName(startAfter string, objects []*data.ObjectInfo) []*data.ObjectInfo { if len(objects) != 0 && objects[len(objects)-1].Name <= startAfter { return nil } @@ -552,9 +553,9 @@ func trimAfterObjectName(startAfter string, objects []*api.ObjectInfo) []*api.Ob return nil } -func trimAfterObjectID(id string, objects []*api.ObjectInfo) []*api.ObjectInfo { +func trimAfterObjectID(id string, objects []*data.ObjectInfo) []*data.ObjectInfo { if len(objects) != 0 && objects[len(objects)-1].ID.String() == id { - return []*api.ObjectInfo{} + return []*data.ObjectInfo{} } for i, obj := range objects { if obj.ID.String() == id { @@ -565,7 +566,7 @@ func trimAfterObjectID(id string, objects []*api.ObjectInfo) []*api.ObjectInfo { return nil } -func triageObjects(allObjects []*api.ObjectInfo) (prefixes []string, objects []*api.ObjectInfo) { +func triageObjects(allObjects []*data.ObjectInfo) (prefixes []string, objects []*data.ObjectInfo) { for _, ov := range allObjects { if ov.IsDir { prefixes = append(prefixes, ov.Name) @@ -577,11 +578,11 @@ func triageObjects(allObjects []*api.ObjectInfo) (prefixes []string, objects []* return } -func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*api.ObjectInfo, error) { +func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*data.ObjectInfo, error) { var ( err error - bkt *api.BucketInfo - allObjects []*api.ObjectInfo + bkt *data.BucketInfo + allObjects []*data.ObjectInfo ) if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { @@ -600,7 +601,7 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ( return allObjects, nil } -func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *api.BucketInfo) bool { +func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *data.BucketInfo) bool { settings, err := n.getBucketSettings(ctx, bktInfo) if err != nil { n.log.Warn("couldn't get versioning settings object", zap.Error(err)) diff --git a/api/layer/object_test.go b/api/layer/object_test.go index 4e90fd7..8bfa69a 100644 --- a/api/layer/object_test.go +++ b/api/layer/object_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/stretchr/testify/require" ) @@ -26,11 +26,11 @@ func randSHA256Checksum(t *testing.T) (cs [sha256.Size]byte) { func TestTrimAfterObjectName(t *testing.T) { var ( - objects []*api.ObjectInfo + objects []*data.ObjectInfo names = []string{"b", "c", "d"} ) for _, name := range names { - objects = append(objects, &api.ObjectInfo{Name: name}) + objects = append(objects, &data.ObjectInfo{Name: name}) } t.Run("startafter before all objects", func(t *testing.T) { @@ -59,7 +59,7 @@ func TestTrimAfterObjectName(t *testing.T) { }) t.Run("empty objects", func(t *testing.T) { - actual := trimAfterObjectName(names[0], []*api.ObjectInfo{}) + actual := trimAfterObjectName(names[0], []*data.ObjectInfo{}) require.Nil(t, actual) }) @@ -76,14 +76,14 @@ func TestTrimAfterObjectName(t *testing.T) { func TestTrimAfterObjectID(t *testing.T) { var ( - objects []*api.ObjectInfo + objects []*data.ObjectInfo ids []*object.ID numberOfIDS = 3 ) for i := 0; i < numberOfIDS; i++ { id := randID(t) - objects = append(objects, &api.ObjectInfo{ID: id}) + objects = append(objects, &data.ObjectInfo{ID: id}) ids = append(ids, id) } diff --git a/api/layer/util.go b/api/layer/util.go index f6c3d8b..09fc903 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -10,6 +10,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" ) @@ -17,7 +18,7 @@ type ( // ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2. ListObjectsInfo struct { Prefixes []string - Objects []*api.ObjectInfo + Objects []*data.ObjectInfo IsTruncated bool } @@ -35,7 +36,7 @@ type ( // ObjectVersionInfo stores info about objects versions. ObjectVersionInfo struct { - Object *api.ObjectInfo + Object *data.ObjectInfo IsLatest bool } @@ -65,11 +66,11 @@ func userHeaders(attrs []*object.Attribute) map[string]string { return result } -func objInfoFromMeta(bkt *api.BucketInfo, meta *object.Object) *api.ObjectInfo { +func objInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectInfo { return objectInfoFromMeta(bkt, meta, "", "") } -func objectInfoFromMeta(bkt *api.BucketInfo, meta *object.Object, prefix, delimiter string) *api.ObjectInfo { +func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object, prefix, delimiter string) *data.ObjectInfo { var ( isDir bool size int64 @@ -110,7 +111,7 @@ func objectInfoFromMeta(bkt *api.BucketInfo, meta *object.Object, prefix, delimi size = int64(meta.PayloadSize()) } - return &api.ObjectInfo{ + return &data.ObjectInfo{ ID: meta.ID(), CID: bkt.CID, IsDir: isDir, diff --git a/api/layer/util_test.go b/api/layer/util_test.go index d06b46f..f738d65 100644 --- a/api/layer/util_test.go +++ b/api/layer/util_test.go @@ -9,7 +9,7 @@ import ( cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" - "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/stretchr/testify/require" ) @@ -20,7 +20,7 @@ var ( defaultTestContentType = http.DetectContentType(defaultTestPayload) ) -func newTestObject(oid *object.ID, bkt *api.BucketInfo, name string) *object.Object { +func newTestObject(oid *object.ID, bkt *data.BucketInfo, name string) *object.Object { filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) filename.SetValue(name) @@ -44,8 +44,8 @@ func newTestObject(oid *object.ID, bkt *api.BucketInfo, name string) *object.Obj return raw.Object() } -func newTestInfo(oid *object.ID, bkt *api.BucketInfo, name string, isDir bool) *api.ObjectInfo { - info := &api.ObjectInfo{ +func newTestInfo(oid *object.ID, bkt *data.BucketInfo, name string, isDir bool) *data.ObjectInfo { + info := &data.ObjectInfo{ ID: oid, Name: name, Bucket: bkt.Name, @@ -72,7 +72,7 @@ func Test_objectInfoFromMeta(t *testing.T) { oid := object.NewID() containerID := cid.New() - bkt := &api.BucketInfo{ + bkt := &data.BucketInfo{ Name: "test-container", CID: containerID, Owner: uid, @@ -82,7 +82,7 @@ func Test_objectInfoFromMeta(t *testing.T) { cases := []struct { name string prefix string - result *api.ObjectInfo + result *data.ObjectInfo object *object.Object delimiter string }{ diff --git a/api/layer/versioning.go b/api/layer/versioning.go index 9023f3e..400f06f 100644 --- a/api/layer/versioning.go +++ b/api/layer/versioning.go @@ -8,13 +8,13 @@ import ( "strings" "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" ) type objectVersions struct { name string - objects []*api.ObjectInfo + objects []*data.ObjectInfo addList []string delList []string isSorted bool @@ -40,7 +40,7 @@ func (v *objectVersions) isAddListEmpty() bool { return len(v.addList) == 0 } -func (v *objectVersions) appendVersion(oi *api.ObjectInfo) { +func (v *objectVersions) appendVersion(oi *data.ObjectInfo) { delVers := splitVersions(oi.Headers[versionsDelAttr]) v.objects = append(v.objects, oi) @@ -100,7 +100,7 @@ func (v *objectVersions) formAddList() { } } -func containsVersions(obj *api.ObjectInfo, versions []string) bool { +func containsVersions(obj *data.ObjectInfo, versions []string) bool { header := obj.Headers[versionsAddAttr] for _, version := range versions { if !strings.Contains(header, version) { @@ -156,7 +156,7 @@ LOOP: return commonAddedVersions, prevVersions, currentVersions } -func (v *objectVersions) getLast() *api.ObjectInfo { +func (v *objectVersions) getLast() *data.ObjectInfo { if v == nil || len(v.objects) == 0 { return nil } @@ -189,14 +189,14 @@ func (v *objectVersions) existedVersions() []string { return res } -func (v *objectVersions) getFiltered(reverse bool) []*api.ObjectInfo { +func (v *objectVersions) getFiltered(reverse bool) []*data.ObjectInfo { if len(v.objects) == 0 { return nil } v.sort() existedVersions := v.existedVersions() - res := make([]*api.ObjectInfo, 0, len(v.objects)) + res := make([]*data.ObjectInfo, 0, len(v.objects)) for _, version := range v.objects { delMark := version.Headers[versionsDeleteMarkAttr] @@ -223,7 +223,7 @@ func (v *objectVersions) getDelHeader() string { return strings.Join(v.delList, ",") } -func (v *objectVersions) getVersion(oid *object.ID) *api.ObjectInfo { +func (v *objectVersions) getVersion(oid *object.ID) *data.ObjectInfo { for _, version := range v.objects { if version.Version() == oid.String() { if contains(v.delList, oid.String()) { @@ -234,7 +234,7 @@ func (v *objectVersions) getVersion(oid *object.ID) *api.ObjectInfo { } return nil } -func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*api.ObjectInfo, error) { +func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*data.ObjectInfo, error) { bktInfo, err := n.GetBucketInfo(ctx, p.Bucket) if err != nil { return nil, err @@ -264,7 +264,7 @@ func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*Bu func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { var ( versions map[string]*objectVersions - allObjects = make([]*api.ObjectInfo, 0, p.MaxKeys) + allObjects = make([]*data.ObjectInfo, 0, p.MaxKeys) res = &ListObjectVersionsInfo{} reverse = true ) @@ -347,7 +347,7 @@ func contains(list []string, elem string) bool { return false } -func (n *layer) getBucketSettings(ctx context.Context, bktInfo *api.BucketInfo) (*BucketSettings, error) { +func (n *layer) getBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*BucketSettings, error) { objInfo, err := n.getSystemObject(ctx, bktInfo, bktInfo.SettingsObjectName()) if err != nil { return nil, err @@ -356,7 +356,7 @@ func (n *layer) getBucketSettings(ctx context.Context, bktInfo *api.BucketInfo) return objectInfoToBucketSettings(objInfo), nil } -func objectInfoToBucketSettings(info *api.ObjectInfo) *BucketSettings { +func objectInfoToBucketSettings(info *data.ObjectInfo) *BucketSettings { res := &BucketSettings{} enabled, ok := info.Headers[attrSettingsVersioningEnabled] @@ -368,7 +368,7 @@ func objectInfoToBucketSettings(info *api.ObjectInfo) *BucketSettings { return res } -func (n *layer) checkVersionsExist(ctx context.Context, bkt *api.BucketInfo, obj *VersionedObject) (*api.ObjectInfo, error) { +func (n *layer) checkVersionsExist(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.ObjectInfo, error) { id := object.NewID() if err := id.Parse(obj.VersionID); err != nil { return nil, errors.GetAPIError(errors.ErrInvalidVersion) diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go index 0fdbb10..ffe1430 100644 --- a/api/layer/versioning_test.go +++ b/api/layer/versioning_test.go @@ -21,7 +21,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/session" "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-s3-gw/api" - "github.com/nspcc-dev/neofs-s3-gw/api/cache" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/nspcc-dev/neofs-sdk-go/pool" @@ -208,7 +208,7 @@ func (t *testPool) WaitForContainerPresence(ctx context.Context, id *cid.ID, par return nil } -func (tc *testContext) putObject(content []byte) *api.ObjectInfo { +func (tc *testContext) putObject(content []byte) *data.ObjectInfo { objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{ Bucket: tc.bkt, Object: tc.obj, @@ -221,7 +221,7 @@ func (tc *testContext) putObject(content []byte) *api.ObjectInfo { return objInfo } -func (tc *testContext) getObject(objectName, versionID string, needError bool) (*api.ObjectInfo, []byte) { +func (tc *testContext) getObject(objectName, versionID string, needError bool) (*data.ObjectInfo, []byte) { objInfo, err := tc.layer.GetObjectInfo(tc.ctx, &HeadObjectParams{ Bucket: tc.bkt, Object: objectName, @@ -254,7 +254,7 @@ func (tc *testContext) deleteObject(objectName, versionID string) { } } -func (tc *testContext) listObjectsV1() []*api.ObjectInfo { +func (tc *testContext) listObjectsV1() []*data.ObjectInfo { res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{ ListObjectsParamsCommon: ListObjectsParamsCommon{ Bucket: tc.bkt, @@ -265,7 +265,7 @@ func (tc *testContext) listObjectsV1() []*api.ObjectInfo { return res.Objects } -func (tc *testContext) listObjectsV2() []*api.ObjectInfo { +func (tc *testContext) listObjectsV2() []*data.ObjectInfo { res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{ ListObjectsParamsCommon: ListObjectsParamsCommon{ Bucket: tc.bkt, @@ -329,13 +329,8 @@ func prepareContext(t *testing.T) *testContext { require.NoError(t, err) return &testContext{ - ctx: ctx, - layer: NewLayer(l, tp, &CacheConfig{ - Size: cache.DefaultObjectsCacheSize, - Lifetime: cache.DefaultObjectsCacheLifetime, - ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime, - ListObjectsSize: cache.DefaultObjectsListCacheSize}, - ), + ctx: ctx, + layer: NewLayer(l, tp, DefaultCachesConfigs()), bkt: bktName, bktID: bktID, obj: "obj1", @@ -457,7 +452,7 @@ func TestGetLastVersion(t *testing.T) { for _, tc := range []struct { versions *objectVersions - expected *api.ObjectInfo + expected *data.ObjectInfo }{ { versions: &objectVersions{}, @@ -465,21 +460,21 @@ func TestGetLastVersion(t *testing.T) { }, { versions: &objectVersions{ - objects: []*api.ObjectInfo{obj2, obj1}, + objects: []*data.ObjectInfo{obj2, obj1}, addList: []string{obj1.Version(), obj2.Version()}, }, expected: obj2, }, { versions: &objectVersions{ - objects: []*api.ObjectInfo{obj2, obj1, obj3}, + objects: []*data.ObjectInfo{obj2, obj1, obj3}, addList: []string{obj1.Version(), obj2.Version(), obj3.Version()}, }, expected: nil, }, { versions: &objectVersions{ - objects: []*api.ObjectInfo{obj2, obj1, obj4}, + objects: []*data.ObjectInfo{obj2, obj1, obj4}, addList: []string{obj1.Version(), obj2.Version(), obj4.Version()}, delList: []string{obj2.Version()}, }, @@ -487,7 +482,7 @@ func TestGetLastVersion(t *testing.T) { }, { versions: &objectVersions{ - objects: []*api.ObjectInfo{obj1, obj5}, + objects: []*data.ObjectInfo{obj1, obj5}, addList: []string{obj1.Version(), obj5.Version()}, delList: []string{obj1.Version()}, }, @@ -495,13 +490,13 @@ func TestGetLastVersion(t *testing.T) { }, { versions: &objectVersions{ - objects: []*api.ObjectInfo{obj5}, + objects: []*data.ObjectInfo{obj5}, }, expected: nil, }, { versions: &objectVersions{ - objects: []*api.ObjectInfo{obj1, obj2, obj3, obj6}, + objects: []*data.ObjectInfo{obj1, obj2, obj3, obj6}, addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj6.Version()}, delList: []string{obj3.Version()}, }, @@ -509,7 +504,7 @@ func TestGetLastVersion(t *testing.T) { }, { versions: &objectVersions{ - objects: []*api.ObjectInfo{obj1, obj1V2}, + objects: []*data.ObjectInfo{obj1, obj1V2}, addList: []string{obj1.Version(), obj1V2.Version()}, }, expected: obj1V2, @@ -530,51 +525,51 @@ func TestAppendVersions(t *testing.T) { for _, tc := range []struct { versions *objectVersions - objectToAdd *api.ObjectInfo + objectToAdd *data.ObjectInfo expectedVersions *objectVersions }{ { versions: &objectVersions{}, objectToAdd: obj1, expectedVersions: &objectVersions{ - objects: []*api.ObjectInfo{obj1}, + objects: []*data.ObjectInfo{obj1}, addList: []string{obj1.Version()}, isSorted: true, }, }, { - versions: &objectVersions{objects: []*api.ObjectInfo{obj1}}, + versions: &objectVersions{objects: []*data.ObjectInfo{obj1}}, objectToAdd: obj2, expectedVersions: &objectVersions{ - objects: []*api.ObjectInfo{obj1, obj2}, + objects: []*data.ObjectInfo{obj1, obj2}, addList: []string{obj1.Version(), obj2.Version()}, isSorted: true, }, }, { - versions: &objectVersions{objects: []*api.ObjectInfo{obj1, obj2}}, + versions: &objectVersions{objects: []*data.ObjectInfo{obj1, obj2}}, objectToAdd: obj3, expectedVersions: &objectVersions{ - objects: []*api.ObjectInfo{obj1, obj2, obj3}, + objects: []*data.ObjectInfo{obj1, obj2, obj3}, addList: []string{obj1.Version(), obj2.Version(), obj3.Version()}, isSorted: true, }, }, { - versions: &objectVersions{objects: []*api.ObjectInfo{obj1, obj2}}, + versions: &objectVersions{objects: []*data.ObjectInfo{obj1, obj2}}, objectToAdd: obj4, expectedVersions: &objectVersions{ - objects: []*api.ObjectInfo{obj1, obj2, obj4}, + objects: []*data.ObjectInfo{obj1, obj2, obj4}, addList: []string{obj1.Version(), obj2.Version(), obj4.Version()}, delList: []string{obj2.Version()}, isSorted: true, }, }, { - versions: &objectVersions{objects: []*api.ObjectInfo{obj5}}, + versions: &objectVersions{objects: []*data.ObjectInfo{obj5}}, objectToAdd: obj6, expectedVersions: &objectVersions{ - objects: []*api.ObjectInfo{obj5, obj6}, + objects: []*data.ObjectInfo{obj5, obj6}, addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj5.Version(), obj6.Version()}, isSorted: true, }, @@ -606,15 +601,15 @@ func TestSortAddHeaders(t *testing.T) { expectedAddHeaders string }{ { - versions: &objectVersions{objects: []*api.ObjectInfo{obj6, obj7, obj8}}, + versions: &objectVersions{objects: []*data.ObjectInfo{obj6, obj7, obj8}}, expectedAddHeaders: joinVers(obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8), }, { - versions: &objectVersions{objects: []*api.ObjectInfo{obj7, obj9}}, + versions: &objectVersions{objects: []*data.ObjectInfo{obj7, obj9}}, expectedAddHeaders: joinVers(obj1, obj4, obj5, obj7, obj9), }, { - versions: &objectVersions{objects: []*api.ObjectInfo{obj11, obj10, obj12}}, + versions: &objectVersions{objects: []*data.ObjectInfo{obj11, obj10, obj12}}, expectedAddHeaders: joinVers(obj10, obj11, obj12), }, } { @@ -622,7 +617,7 @@ func TestSortAddHeaders(t *testing.T) { } } -func joinVers(objs ...*api.ObjectInfo) string { +func joinVers(objs ...*data.ObjectInfo) string { if len(objs) == 0 { return "" } @@ -643,7 +638,7 @@ func getOID(id byte) *object.ID { return oid } -func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *api.ObjectInfo { +func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo { headers := make(map[string]string) if addAttr != "" { headers[versionsAddAttr] = addAttr @@ -655,14 +650,14 @@ func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *api.Objec headers[versionsDeleteMarkAttr] = delMarkAttr } - return &api.ObjectInfo{ + return &data.ObjectInfo{ ID: getOID(id), Name: strconv.Itoa(int(id)), Headers: headers, } } -func getTestObjectInfoEpoch(epoch uint64, id byte, addAttr, delAttr, delMarkAttr string) *api.ObjectInfo { +func getTestObjectInfoEpoch(epoch uint64, id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo { obj := getTestObjectInfo(id, addAttr, delAttr, delMarkAttr) obj.CreationEpoch = epoch return obj diff --git a/authmate/authmate.go b/authmate/authmate.go index e90fa59..a1c8dc5 100644 --- a/authmate/authmate.go +++ b/authmate/authmate.go @@ -22,6 +22,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/session" "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/tokens" "github.com/nspcc-dev/neofs-sdk-go/policy" @@ -226,7 +227,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr } address, err := tokens. - New(a.pool, secrets.EphemeralKey, tokens.DefaultCacheConfig()). + New(a.pool, secrets.EphemeralKey, cache.DefaultAccessBoxConfig()). Put(ctx, cid, oid, box, lifetime.Exp, options.GatesPublicKeys...) if err != nil { return fmt.Errorf("failed to put bearer token: %w", err) @@ -268,7 +269,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr // ObtainSecret receives an existing secret access key from NeoFS and // writes to io.Writer the secret access key. func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error { - bearerCreds := tokens.New(a.pool, options.GatePrivateKey, tokens.DefaultCacheConfig()) + bearerCreds := tokens.New(a.pool, options.GatePrivateKey, cache.DefaultAccessBoxConfig()) address := object.NewAddress() if err := address.Parse(options.SecretAddress); err != nil { return fmt.Errorf("failed to parse secret address: %w", err) diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index 71ba0aa..da4cabc 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -14,7 +14,6 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/api/handler" "github.com/nspcc-dev/neofs-s3-gw/api/layer" - "github.com/nspcc-dev/neofs-s3-gw/creds/tokens" "github.com/nspcc-dev/neofs-s3-gw/internal/wallet" "github.com/nspcc-dev/neofs-sdk-go/policy" "github.com/nspcc-dev/neofs-sdk-go/pool" @@ -216,21 +215,25 @@ func (a *App) Server(ctx context.Context) { close(a.webDone) } -func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig { - cacheCfg := layer.CacheConfig{ - ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime, - ListObjectsSize: cache.DefaultObjectsListCacheSize, - Size: cache.DefaultObjectsCacheSize, - Lifetime: cache.DefaultObjectsCacheLifetime, - } +func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CachesConfig { + cacheCfg := layer.DefaultCachesConfigs() - cacheCfg.Lifetime = getLifetime(v, l, cfgObjectsCacheLifetime, cacheCfg.Lifetime) - cacheCfg.Size = getSize(v, l, cfgObjectsCacheSize, cacheCfg.Size) + cacheCfg.Objects.Lifetime = getLifetime(v, l, cfgObjectsCacheLifetime, cacheCfg.Objects.Lifetime) + cacheCfg.Objects.Size = getSize(v, l, cfgObjectsCacheSize, cacheCfg.Objects.Size) - cacheCfg.ListObjectsLifetime = getLifetime(v, l, cfgListObjectsCacheLifetime, cacheCfg.ListObjectsLifetime) - cacheCfg.ListObjectsSize = getSize(v, l, cfgListObjectsCacheSize, cacheCfg.ListObjectsSize) + cacheCfg.ObjectsList.Lifetime = getLifetime(v, l, cfgListObjectsCacheLifetime, cacheCfg.ObjectsList.Lifetime) + cacheCfg.ObjectsList.Size = getSize(v, l, cfgListObjectsCacheSize, cacheCfg.ObjectsList.Size) - return &cacheCfg + cacheCfg.Buckets.Lifetime = getLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Buckets.Lifetime) + cacheCfg.Buckets.Size = getSize(v, l, cfgBucketsCacheSize, cacheCfg.Buckets.Size) + + cacheCfg.Names.Lifetime = getLifetime(v, l, cfgNamesCacheLifetime, cacheCfg.Names.Lifetime) + cacheCfg.Names.Size = getSize(v, l, cfgNamesCacheSize, cacheCfg.Names.Size) + + cacheCfg.System.Lifetime = getLifetime(v, l, cfgSystemLifetimeSize, cacheCfg.System.Lifetime) + cacheCfg.System.Size = getSize(v, l, cfgSystemCacheSize, cacheCfg.System.Size) + + return cacheCfg } func getLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration { @@ -263,8 +266,8 @@ func getSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue int) i return defaultValue } -func getAccessBoxCacheConfig(v *viper.Viper, l *zap.Logger) *tokens.CacheConfig { - cacheCfg := tokens.DefaultCacheConfig() +func getAccessBoxCacheConfig(v *viper.Viper, l *zap.Logger) *cache.Config { + cacheCfg := cache.DefaultAccessBoxConfig() cacheCfg.Lifetime = getLifetime(v, l, cfgAccessBoxCacheLifetime, cacheCfg.Lifetime) cacheCfg.Size = getSize(v, l, cfgAccessBoxCacheSize, cacheCfg.Size) diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go index 308b851..d4630f4 100644 --- a/cmd/s3-gw/app_settings.go +++ b/cmd/s3-gw/app_settings.go @@ -59,6 +59,12 @@ const ( // Settings. cfgObjectsCacheSize = "cache.objects.size" cfgListObjectsCacheLifetime = "cache.list.lifetime" cfgListObjectsCacheSize = "cache.list.size" + cfgBucketsCacheLifetime = "cache.buckets.lifetime" + cfgBucketsCacheSize = "cache.buckets.size" + cfgNamesCacheLifetime = "cache.names.lifetime" + cfgNamesCacheSize = "cache.names.size" + cfgSystemLifetimeSize = "cache.system.lifetime" + cfgSystemCacheSize = "cache.system.size" cfgAccessBoxCacheLifetime = "cache.accessbox.lifetime" cfgAccessBoxCacheSize = "cache.accessbox.size" diff --git a/creds/tokens/credentials.go b/creds/tokens/credentials.go index ce3ae36..203b6bd 100644 --- a/creds/tokens/credentials.go +++ b/creds/tokens/credentials.go @@ -13,6 +13,7 @@ import ( cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-sdk-go/pool" ) @@ -27,7 +28,7 @@ type ( cred struct { key *keys.PrivateKey pool pool.Pool - cache *AccessBoxCache + cache *cache.AccessBoxCache } ) @@ -47,8 +48,8 @@ var bufferPool = sync.Pool{ var _ = New // New creates new Credentials instance using given cli and key. -func New(conns pool.Pool, key *keys.PrivateKey, config *CacheConfig) Credentials { - return &cred{pool: conns, key: key, cache: NewAccessBoxCache(config)} +func New(conns pool.Pool, key *keys.PrivateKey, config *cache.Config) Credentials { + return &cred{pool: conns, key: key, cache: cache.NewAccessBoxCache(config)} } func (c *cred) acquireBuffer() *bytes.Buffer { diff --git a/docs/configuration.md b/docs/configuration.md index fbd997d..0bff1df 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -114,6 +114,15 @@ cache: list: lifetime: 1m size: 100 + names: + lifetime: 1m + size: 1000 + buckets: + lifetime: 1m + size: 500 + system: + lifetime: 2m + size: 1000 accessbox: lifetime: 5m size: 10