forked from TrueCloudLab/frostfs-s3-gw
[#253] Caches refactoring
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
parent
951eb6fda8
commit
19b917e3b5
30 changed files with 365 additions and 322 deletions
|
@ -18,6 +18,7 @@ import (
|
||||||
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
||||||
|
@ -81,7 +82,7 @@ func (p prs) Seek(_ int64, _ int) (int64, error) {
|
||||||
var _ io.ReadSeeker = prs(0)
|
var _ io.ReadSeeker = prs(0)
|
||||||
|
|
||||||
// New creates an instance of AuthCenter.
|
// New creates an instance of AuthCenter.
|
||||||
func New(conns pool.Pool, key *keys.PrivateKey, config *tokens.CacheConfig) Center {
|
func New(conns pool.Pool, key *keys.PrivateKey, config *cache.Config) Center {
|
||||||
return ¢er{
|
return ¢er{
|
||||||
cli: tokens.New(conns, key, config),
|
cli: tokens.New(conns, key, config),
|
||||||
reg: ®expSubmatcher{re: authorizationFieldRegexp},
|
reg: ®expSubmatcher{re: authorizationFieldRegexp},
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package tokens
|
package cache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
@ -8,31 +8,33 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
type (
|
||||||
// DefaultCacheSize is a default maximum number of entries in cache.
|
// AccessBoxCache stores access box by its address.
|
||||||
DefaultCacheSize = 100
|
AccessBoxCache struct {
|
||||||
// DefaultCacheLifetime is a default lifetime of entries in cache.
|
cache gcache.Cache
|
||||||
DefaultCacheLifetime = 10 * time.Minute
|
}
|
||||||
)
|
|
||||||
|
|
||||||
// CacheConfig stores expiration params for cache.
|
// Config stores expiration params for cache.
|
||||||
type CacheConfig struct {
|
Config struct {
|
||||||
Size int
|
Size int
|
||||||
Lifetime time.Duration
|
Lifetime time.Duration
|
||||||
}
|
}
|
||||||
|
)
|
||||||
|
|
||||||
// DefaultCacheConfig return new default cache expiration values.
|
const (
|
||||||
func DefaultCacheConfig() *CacheConfig {
|
// DefaultAccessBoxCacheSize is a default maximum number of entries in cache.
|
||||||
return &CacheConfig{Size: DefaultCacheSize, Lifetime: DefaultCacheLifetime}
|
DefaultAccessBoxCacheSize = 100
|
||||||
}
|
// DefaultAccessBoxCacheLifetime is a default lifetime of entries in cache.
|
||||||
|
DefaultAccessBoxCacheLifetime = 10 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
// AccessBoxCache stores access box by its address.
|
// DefaultAccessBoxConfig return new default cache expiration values.
|
||||||
type AccessBoxCache struct {
|
func DefaultAccessBoxConfig() *Config {
|
||||||
cache gcache.Cache
|
return &Config{Size: DefaultAccessBoxCacheSize, Lifetime: DefaultAccessBoxCacheLifetime}
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewAccessBoxCache creates an object of BucketCache.
|
// NewAccessBoxCache creates an object of BucketCache.
|
||||||
func NewAccessBoxCache(config *CacheConfig) *AccessBoxCache {
|
func NewAccessBoxCache(config *Config) *AccessBoxCache {
|
||||||
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||||
|
|
||||||
return &AccessBoxCache{cache: gc}
|
return &AccessBoxCache{cache: gc}
|
45
api/cache/buckets.go
vendored
45
api/cache/buckets.go
vendored
|
@ -4,39 +4,40 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
// BucketCache contains cache with objects and lifetime of cache entries.
|
||||||
// BucketCache provides interface for lru cache for objects.
|
type BucketCache struct {
|
||||||
BucketCache interface {
|
|
||||||
Get(key string) *api.BucketInfo
|
|
||||||
Put(bkt *api.BucketInfo) error
|
|
||||||
Delete(key string) bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetBucketCache contains cache with objects and lifetime of cache entries.
|
|
||||||
GetBucketCache struct {
|
|
||||||
cache gcache.Cache
|
cache gcache.Cache
|
||||||
lifetime time.Duration
|
}
|
||||||
}
|
|
||||||
|
const (
|
||||||
|
// DefaultBucketCacheSize is a default maximum number of entries in cache.
|
||||||
|
DefaultBucketCacheSize = 1e3
|
||||||
|
// DefaultBucketCacheLifetime is a default lifetime of entries in cache.
|
||||||
|
DefaultBucketCacheLifetime = time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// DefaultBucketConfig return new default cache expiration values.
|
||||||
|
func DefaultBucketConfig() *Config {
|
||||||
|
return &Config{Size: DefaultBucketCacheSize, Lifetime: DefaultBucketCacheLifetime}
|
||||||
|
}
|
||||||
|
|
||||||
// NewBucketCache creates an object of BucketCache.
|
// NewBucketCache creates an object of BucketCache.
|
||||||
func NewBucketCache(cacheSize int, lifetime time.Duration) *GetBucketCache {
|
func NewBucketCache(config *Config) *BucketCache {
|
||||||
gc := gcache.New(cacheSize).LRU().Build()
|
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||||
|
return &BucketCache{cache: gc}
|
||||||
return &GetBucketCache{cache: gc, lifetime: lifetime}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns cached object.
|
// Get returns cached object.
|
||||||
func (o *GetBucketCache) Get(key string) *api.BucketInfo {
|
func (o *BucketCache) Get(key string) *data.BucketInfo {
|
||||||
entry, err := o.cache.Get(key)
|
entry, err := o.cache.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
result, ok := entry.(*api.BucketInfo)
|
result, ok := entry.(*data.BucketInfo)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -45,11 +46,11 @@ func (o *GetBucketCache) Get(key string) *api.BucketInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put puts an object to cache.
|
// Put puts an object to cache.
|
||||||
func (o *GetBucketCache) Put(bkt *api.BucketInfo) error {
|
func (o *BucketCache) Put(bkt *data.BucketInfo) error {
|
||||||
return o.cache.SetWithExpire(bkt.Name, bkt, o.lifetime)
|
return o.cache.Set(bkt.Name, bkt)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete deletes an object from cache.
|
// Delete deletes an object from cache.
|
||||||
func (o *GetBucketCache) Delete(key string) bool {
|
func (o *BucketCache) Delete(key string) bool {
|
||||||
return o.cache.Remove(key)
|
return o.cache.Remove(key)
|
||||||
}
|
}
|
||||||
|
|
39
api/cache/names.go
vendored
39
api/cache/names.go
vendored
|
@ -7,32 +7,33 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ObjectsNameCache provides interface for lru cache for objects.
|
// ObjectsNameCache provides for lru cache for objects.
|
||||||
// This cache contains mapping nice name to object addresses.
|
// This cache contains mapping nice name to object addresses.
|
||||||
// Key is bucketName+objectName.
|
// Key is bucketName+objectName.
|
||||||
type ObjectsNameCache interface {
|
type ObjectsNameCache struct {
|
||||||
Get(key string) *object.Address
|
cache gcache.Cache
|
||||||
Put(key string, address *object.Address) error
|
|
||||||
Delete(key string) bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type (
|
const (
|
||||||
// NameCache contains cache with objects and lifetime of cache entries.
|
// DefaultObjectsNameCacheSize is a default maximum number of entries in cache.
|
||||||
NameCache struct {
|
DefaultObjectsNameCacheSize = 1e4
|
||||||
cache gcache.Cache
|
// DefaultObjectsNameCacheLifetime is a default lifetime of entries in cache.
|
||||||
lifetime time.Duration
|
DefaultObjectsNameCacheLifetime = time.Minute
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewObjectsNameCache creates an object of ObjectsNameCache.
|
// DefaultObjectsNameConfig return new default cache expiration values.
|
||||||
func NewObjectsNameCache(cacheSize int, lifetime time.Duration) *NameCache {
|
func DefaultObjectsNameConfig() *Config {
|
||||||
gc := gcache.New(cacheSize).LRU().Build()
|
return &Config{Size: DefaultObjectsNameCacheSize, Lifetime: DefaultObjectsNameCacheLifetime}
|
||||||
|
}
|
||||||
|
|
||||||
return &NameCache{cache: gc, lifetime: lifetime}
|
// NewObjectsNameCache creates an object of ObjectsNameCache.
|
||||||
|
func NewObjectsNameCache(config *Config) *ObjectsNameCache {
|
||||||
|
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||||
|
return &ObjectsNameCache{cache: gc}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns cached object.
|
// Get returns cached object.
|
||||||
func (o *NameCache) Get(key string) *object.Address {
|
func (o *ObjectsNameCache) Get(key string) *object.Address {
|
||||||
entry, err := o.cache.Get(key)
|
entry, err := o.cache.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
|
@ -47,11 +48,11 @@ func (o *NameCache) Get(key string) *object.Address {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put puts an object to cache.
|
// Put puts an object to cache.
|
||||||
func (o *NameCache) Put(key string, address *object.Address) error {
|
func (o *ObjectsNameCache) Put(key string, address *object.Address) error {
|
||||||
return o.cache.SetWithExpire(key, address, o.lifetime)
|
return o.cache.Set(key, address)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete deletes an object from cache.
|
// Delete deletes an object from cache.
|
||||||
func (o *NameCache) Delete(key string) bool {
|
func (o *ObjectsNameCache) Delete(key string) bool {
|
||||||
return o.cache.Remove(key)
|
return o.cache.Remove(key)
|
||||||
}
|
}
|
||||||
|
|
34
api/cache/objects.go
vendored
34
api/cache/objects.go
vendored
|
@ -7,11 +7,9 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ObjectsCache provides interface for lru cache for objects.
|
// ObjectsCache provides lru cache for objects.
|
||||||
type ObjectsCache interface {
|
type ObjectsCache struct {
|
||||||
Get(address *object.Address) *object.Object
|
cache gcache.Cache
|
||||||
Put(obj object.Object) error
|
|
||||||
Delete(address *object.Address) bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -21,23 +19,19 @@ const (
|
||||||
DefaultObjectsCacheSize = 1e6
|
DefaultObjectsCacheSize = 1e6
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
// DefaultObjectsConfig return new default cache expiration values.
|
||||||
// ObjectHeadersCache contains cache with objects and lifetime of cache entries.
|
func DefaultObjectsConfig() *Config {
|
||||||
ObjectHeadersCache struct {
|
return &Config{Size: DefaultObjectsCacheSize, Lifetime: DefaultObjectsCacheLifetime}
|
||||||
cache gcache.Cache
|
}
|
||||||
lifetime time.Duration
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
// New creates an object of ObjectHeadersCache.
|
// New creates an object of ObjectHeadersCache.
|
||||||
func New(cacheSize int, lifetime time.Duration) *ObjectHeadersCache {
|
func New(config *Config) *ObjectsCache {
|
||||||
gc := gcache.New(cacheSize).LRU().Build()
|
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||||
|
return &ObjectsCache{cache: gc}
|
||||||
return &ObjectHeadersCache{cache: gc, lifetime: lifetime}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns cached object.
|
// Get returns cached object.
|
||||||
func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object {
|
func (o *ObjectsCache) Get(address *object.Address) *object.Object {
|
||||||
entry, err := o.cache.Get(address.String())
|
entry, err := o.cache.Get(address.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
|
@ -52,11 +46,11 @@ func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put puts an object to cache.
|
// Put puts an object to cache.
|
||||||
func (o *ObjectHeadersCache) Put(obj object.Object) error {
|
func (o *ObjectsCache) Put(obj object.Object) error {
|
||||||
return o.cache.SetWithExpire(obj.ContainerID().String()+"/"+obj.ID().String(), obj, o.lifetime)
|
return o.cache.Set(obj.ContainerID().String()+"/"+obj.ID().String(), obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete deletes an object from cache.
|
// Delete deletes an object from cache.
|
||||||
func (o *ObjectHeadersCache) Delete(address *object.Address) bool {
|
func (o *ObjectsCache) Delete(address *object.Address) bool {
|
||||||
return o.cache.Remove(address.String())
|
return o.cache.Remove(address.String())
|
||||||
}
|
}
|
||||||
|
|
14
api/cache/objects_test.go
vendored
14
api/cache/objects_test.go
vendored
|
@ -9,10 +9,12 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
func getTestConfig() *Config {
|
||||||
cachesize = 10
|
return &Config{
|
||||||
lifetime = time.Second * 5
|
Size: 10,
|
||||||
)
|
Lifetime: 5 * time.Second,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCache(t *testing.T) {
|
func TestCache(t *testing.T) {
|
||||||
obj := objecttest.Object()
|
obj := objecttest.Object()
|
||||||
|
@ -21,7 +23,7 @@ func TestCache(t *testing.T) {
|
||||||
address.SetObjectID(obj.ID())
|
address.SetObjectID(obj.ID())
|
||||||
|
|
||||||
t.Run("check get", func(t *testing.T) {
|
t.Run("check get", func(t *testing.T) {
|
||||||
cache := New(cachesize, lifetime)
|
cache := New(getTestConfig())
|
||||||
err := cache.Put(*obj)
|
err := cache.Put(*obj)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -30,7 +32,7 @@ func TestCache(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("check delete", func(t *testing.T) {
|
t.Run("check delete", func(t *testing.T) {
|
||||||
cache := New(cachesize, lifetime)
|
cache := New(getTestConfig())
|
||||||
err := cache.Put(*obj)
|
err := cache.Put(*obj)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
|
51
api/cache/objectslist.go
vendored
51
api/cache/objectslist.go
vendored
|
@ -24,25 +24,8 @@ import (
|
||||||
*/
|
*/
|
||||||
|
|
||||||
type (
|
type (
|
||||||
// ObjectsListCache provides interface for cache of ListObjectsV2 in a layer struct.
|
// ObjectsListCache contains cache for ListObjects and ListObjectVersions.
|
||||||
ObjectsListCache interface {
|
ObjectsListCache struct {
|
||||||
Get(key ObjectsListKey) []*object.ID
|
|
||||||
Put(key ObjectsListKey, oids []*object.ID) error
|
|
||||||
CleanCacheEntriesContainingObject(objectName string, cid *cid.ID)
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
|
|
||||||
DefaultObjectsListCacheLifetime = time.Second * 60
|
|
||||||
// DefaultObjectsListCacheSize is a default size of cache of ListObjects.
|
|
||||||
DefaultObjectsListCacheSize = 1e5
|
|
||||||
)
|
|
||||||
|
|
||||||
type (
|
|
||||||
// ListObjectsCache contains cache for ListObjects and ListObjectVersions.
|
|
||||||
ListObjectsCache struct {
|
|
||||||
lifetime time.Duration
|
|
||||||
cache gcache.Cache
|
cache gcache.Cache
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -53,18 +36,26 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewObjectsListCache is a constructor which creates an object of ListObjectsCache with given lifetime of entries.
|
const (
|
||||||
func NewObjectsListCache(cacheSize int, lifetime time.Duration) *ListObjectsCache {
|
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
|
||||||
gc := gcache.New(cacheSize).LRU().Build()
|
DefaultObjectsListCacheLifetime = time.Second * 60
|
||||||
|
// DefaultObjectsListCacheSize is a default size of cache of ListObjects.
|
||||||
|
DefaultObjectsListCacheSize = 1e5
|
||||||
|
)
|
||||||
|
|
||||||
return &ListObjectsCache{
|
// DefaultObjectsListConfig return new default cache expiration values.
|
||||||
cache: gc,
|
func DefaultObjectsListConfig() *Config {
|
||||||
lifetime: lifetime,
|
return &Config{Size: DefaultObjectsListCacheSize, Lifetime: DefaultObjectsListCacheLifetime}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewObjectsListCache is a constructor which creates an object of ListObjectsCache with given lifetime of entries.
|
||||||
|
func NewObjectsListCache(config *Config) *ObjectsListCache {
|
||||||
|
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||||
|
return &ObjectsListCache{cache: gc}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get return list of ObjectInfo.
|
// Get return list of ObjectInfo.
|
||||||
func (l *ListObjectsCache) Get(key ObjectsListKey) []*object.ID {
|
func (l *ObjectsListCache) Get(key ObjectsListKey) []*object.ID {
|
||||||
entry, err := l.cache.Get(key)
|
entry, err := l.cache.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
|
@ -79,16 +70,16 @@ func (l *ListObjectsCache) Get(key ObjectsListKey) []*object.ID {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put puts a list of objects to cache.
|
// Put puts a list of objects to cache.
|
||||||
func (l *ListObjectsCache) Put(key ObjectsListKey, oids []*object.ID) error {
|
func (l *ObjectsListCache) Put(key ObjectsListKey, oids []*object.ID) error {
|
||||||
if len(oids) == 0 {
|
if len(oids) == 0 {
|
||||||
return fmt.Errorf("list is empty, cid: %s, prefix: %s", key.cid, key.prefix)
|
return fmt.Errorf("list is empty, cid: %s, prefix: %s", key.cid, key.prefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
return l.cache.SetWithExpire(key, oids, l.lifetime)
|
return l.cache.Set(key, oids)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanCacheEntriesContainingObject deletes entries containing specified object.
|
// CleanCacheEntriesContainingObject deletes entries containing specified object.
|
||||||
func (l *ListObjectsCache) CleanCacheEntriesContainingObject(objectName string, cid *cid.ID) {
|
func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string, cid *cid.ID) {
|
||||||
cidStr := cid.String()
|
cidStr := cid.String()
|
||||||
keys := l.cache.Keys(true)
|
keys := l.cache.Keys(true)
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
|
|
32
api/cache/objectslist_test.go
vendored
32
api/cache/objectslist_test.go
vendored
|
@ -14,6 +14,13 @@ import (
|
||||||
const testingCacheLifetime = 5 * time.Second
|
const testingCacheLifetime = 5 * time.Second
|
||||||
const testingCacheSize = 10
|
const testingCacheSize = 10
|
||||||
|
|
||||||
|
func getTestObjectsListConfig() *Config {
|
||||||
|
return &Config{
|
||||||
|
Size: testingCacheSize,
|
||||||
|
Lifetime: testingCacheLifetime,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func randID(t *testing.T) *object.ID {
|
func randID(t *testing.T) *object.ID {
|
||||||
id := object.NewID()
|
id := object.NewID()
|
||||||
id.SetSHA256(randSHA256Checksum(t))
|
id.SetSHA256(randSHA256Checksum(t))
|
||||||
|
@ -42,7 +49,8 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
|
|
||||||
t.Run("lifetime", func(t *testing.T) {
|
t.Run("lifetime", func(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
cache = NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
config = getTestObjectsListConfig()
|
||||||
|
cache = NewObjectsListCache(config)
|
||||||
cacheKey = ObjectsListKey{cid: userKey}
|
cacheKey = ObjectsListKey{cid: userKey}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -53,13 +61,13 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
return cache.Get(cacheKey) == nil
|
return cache.Get(cacheKey) == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
require.Never(t, condition, cache.lifetime, time.Second)
|
require.Never(t, condition, config.Lifetime, time.Second)
|
||||||
require.Eventually(t, condition, time.Second, 10*time.Millisecond)
|
require.Eventually(t, condition, time.Second, 10*time.Millisecond)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("get cache with empty prefix", func(t *testing.T) {
|
t.Run("get cache with empty prefix", func(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
cache = NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
cache = NewObjectsListCache(getTestObjectsListConfig())
|
||||||
cacheKey = ObjectsListKey{cid: userKey}
|
cacheKey = ObjectsListKey{cid: userKey}
|
||||||
)
|
)
|
||||||
err := cache.Put(cacheKey, ids)
|
err := cache.Put(cacheKey, ids)
|
||||||
|
@ -79,7 +87,7 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
prefix: "dir",
|
prefix: "dir",
|
||||||
}
|
}
|
||||||
|
|
||||||
cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
cache := NewObjectsListCache(getTestObjectsListConfig())
|
||||||
err := cache.Put(cacheKey, ids)
|
err := cache.Put(cacheKey, ids)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -104,7 +112,7 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
cache := NewObjectsListCache(getTestObjectsListConfig())
|
||||||
err := cache.Put(cacheKey, ids)
|
err := cache.Put(cacheKey, ids)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -122,7 +130,7 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
cache := NewObjectsListCache(getTestObjectsListConfig())
|
||||||
err := cache.Put(cacheKey, ids)
|
err := cache.Put(cacheKey, ids)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -143,7 +151,9 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("put object to the root of the bucket", func(t *testing.T) {
|
t.Run("put object to the root of the bucket", func(t *testing.T) {
|
||||||
cache := NewObjectsListCache(testingCacheSize, time.Minute)
|
config := getTestObjectsListConfig()
|
||||||
|
config.Lifetime = time.Minute
|
||||||
|
cache := NewObjectsListCache(config)
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
err := cache.Put(k, oids)
|
err := cache.Put(k, oids)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -160,7 +170,9 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("put object to dir/", func(t *testing.T) {
|
t.Run("put object to dir/", func(t *testing.T) {
|
||||||
cache := NewObjectsListCache(testingCacheSize, time.Minute)
|
config := getTestObjectsListConfig()
|
||||||
|
config.Lifetime = time.Minute
|
||||||
|
cache := NewObjectsListCache(config)
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
err := cache.Put(k, oids)
|
err := cache.Put(k, oids)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -177,7 +189,9 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("put object to dir/lol/", func(t *testing.T) {
|
t.Run("put object to dir/lol/", func(t *testing.T) {
|
||||||
cache := NewObjectsListCache(testingCacheSize, time.Minute)
|
config := getTestObjectsListConfig()
|
||||||
|
config.Lifetime = time.Minute
|
||||||
|
cache := NewObjectsListCache(config)
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
err := cache.Put(k, oids)
|
err := cache.Put(k, oids)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
45
api/cache/system.go
vendored
45
api/cache/system.go
vendored
|
@ -7,32 +7,33 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
// SystemCache provides lru cache for objects.
|
||||||
// SystemCache provides interface for lru cache for objects.
|
// This cache contains "system" objects (bucket versioning settings, tagging object etc.).
|
||||||
// This cache contains "system" objects (bucket versioning settings, tagging object etc.).
|
// Key is bucketName+systemFileName.
|
||||||
// Key is bucketName+systemFileName.
|
type SystemCache struct {
|
||||||
SystemCache interface {
|
|
||||||
Get(key string) *object.Object
|
|
||||||
Put(key string, obj *object.Object) error
|
|
||||||
Delete(key string) bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// SysCache contains cache with objects and lifetime of cache entries.
|
|
||||||
SysCache struct {
|
|
||||||
cache gcache.Cache
|
cache gcache.Cache
|
||||||
lifetime time.Duration
|
}
|
||||||
}
|
|
||||||
|
const (
|
||||||
|
// DefaultSystemCacheSize is a default maximum number of entries in cache.
|
||||||
|
DefaultSystemCacheSize = 1e4
|
||||||
|
// DefaultSystemCacheLifetime is a default lifetime of entries in cache.
|
||||||
|
DefaultSystemCacheLifetime = 5 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewSystemCache creates an object of SystemCache.
|
// DefaultSystemConfig return new default cache expiration values.
|
||||||
func NewSystemCache(cacheSize int, lifetime time.Duration) *SysCache {
|
func DefaultSystemConfig() *Config {
|
||||||
gc := gcache.New(cacheSize).LRU().Build()
|
return &Config{Size: DefaultSystemCacheSize, Lifetime: DefaultSystemCacheLifetime}
|
||||||
|
}
|
||||||
|
|
||||||
return &SysCache{cache: gc, lifetime: lifetime}
|
// NewSystemCache creates an object of SystemCache.
|
||||||
|
func NewSystemCache(config *Config) *SystemCache {
|
||||||
|
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||||
|
return &SystemCache{cache: gc}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns cached object.
|
// Get returns cached object.
|
||||||
func (o *SysCache) Get(key string) *object.Object {
|
func (o *SystemCache) Get(key string) *object.Object {
|
||||||
entry, err := o.cache.Get(key)
|
entry, err := o.cache.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
|
@ -47,11 +48,11 @@ func (o *SysCache) Get(key string) *object.Object {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put puts an object to cache.
|
// Put puts an object to cache.
|
||||||
func (o *SysCache) Put(key string, obj *object.Object) error {
|
func (o *SystemCache) Put(key string, obj *object.Object) error {
|
||||||
return o.cache.SetWithExpire(key, obj, o.lifetime)
|
return o.cache.Set(key, obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete deletes an object from cache.
|
// Delete deletes an object from cache.
|
||||||
func (o *SysCache) Delete(key string) bool {
|
func (o *SystemCache) Delete(key string) bool {
|
||||||
return o.cache.Remove(key)
|
return o.cache.Remove(key)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package api
|
package data
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
|
@ -14,6 +14,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-api-go/v2/acl"
|
"github.com/nspcc-dev/neofs-api-go/v2/acl"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
)
|
)
|
||||||
|
@ -299,7 +300,7 @@ func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkOwner(info *api.BucketInfo, owner string) error {
|
func checkOwner(info *data.BucketInfo, owner string) error {
|
||||||
if owner == "" {
|
if owner == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -32,7 +33,7 @@ func path2BucketObject(path string) (bucket, prefix string) {
|
||||||
func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
info *api.ObjectInfo
|
info *data.ObjectInfo
|
||||||
metadata map[string]string
|
metadata map[string]string
|
||||||
|
|
||||||
reqInfo = api.GetReqInfo(r.Context())
|
reqInfo = api.GetReqInfo(r.Context())
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
)
|
)
|
||||||
|
@ -65,7 +66,7 @@ func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams,
|
||||||
return &layer.RangeParams{Start: start, End: end}, nil
|
return &layer.RangeParams{Start: start, End: end}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeHeaders(h http.Header, info *api.ObjectInfo, tagSetLength int) {
|
func writeHeaders(h http.Header, info *data.ObjectInfo, tagSetLength int) {
|
||||||
if len(info.ContentType) > 0 {
|
if len(info.ContentType) > 0 {
|
||||||
h.Set(api.ContentType, info.ContentType)
|
h.Set(api.ContentType, info.ContentType)
|
||||||
}
|
}
|
||||||
|
@ -83,7 +84,7 @@ func writeHeaders(h http.Header, info *api.ObjectInfo, tagSetLength int) {
|
||||||
func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
info *api.ObjectInfo
|
info *data.ObjectInfo
|
||||||
params *layer.RangeParams
|
params *layer.RangeParams
|
||||||
|
|
||||||
reqInfo = api.GetReqInfo(r.Context())
|
reqInfo = api.GetReqInfo(r.Context())
|
||||||
|
@ -143,7 +144,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkPreconditions(info *api.ObjectInfo, args *conditionalArgs) error {
|
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error {
|
||||||
if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum {
|
if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum {
|
||||||
return errors.GetAPIError(errors.ErrPreconditionFailed)
|
return errors.GetAPIError(errors.ErrPreconditionFailed)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -46,8 +46,8 @@ func TestFetchRangeHeader(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newInfo(etag string, created time.Time) *api.ObjectInfo {
|
func newInfo(etag string, created time.Time) *data.ObjectInfo {
|
||||||
return &api.ObjectInfo{
|
return &data.ObjectInfo{
|
||||||
HashSum: etag,
|
HashSum: etag,
|
||||||
Created: created,
|
Created: created,
|
||||||
}
|
}
|
||||||
|
@ -61,13 +61,13 @@ func TestPreconditions(t *testing.T) {
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
info *api.ObjectInfo
|
info *data.ObjectInfo
|
||||||
args *conditionalArgs
|
args *conditionalArgs
|
||||||
expected error
|
expected error
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "no conditions",
|
name: "no conditions",
|
||||||
info: new(api.ObjectInfo),
|
info: new(data.ObjectInfo),
|
||||||
args: new(conditionalArgs),
|
args: new(conditionalArgs),
|
||||||
expected: nil,
|
expected: nil,
|
||||||
},
|
},
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -27,7 +28,7 @@ func getRangeToDetectContentType(maxSize int64) *layer.RangeParams {
|
||||||
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
info *api.ObjectInfo
|
info *data.ObjectInfo
|
||||||
|
|
||||||
reqInfo = api.GetReqInfo(r.Context())
|
reqInfo = api.GetReqInfo(r.Context())
|
||||||
)
|
)
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
)
|
)
|
||||||
|
@ -183,11 +184,11 @@ func fillPrefixes(src []string, encode string) []CommonPrefix {
|
||||||
return dst
|
return dst
|
||||||
}
|
}
|
||||||
|
|
||||||
func fillContentsWithOwner(src []*api.ObjectInfo, encode string) []Object {
|
func fillContentsWithOwner(src []*data.ObjectInfo, encode string) []Object {
|
||||||
return fillContents(src, encode, true)
|
return fillContents(src, encode, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
func fillContents(src []*api.ObjectInfo, encode string, fetchOwner bool) []Object {
|
func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Object {
|
||||||
var dst []Object
|
var dst []Object
|
||||||
for _, obj := range src {
|
for _, obj := range src {
|
||||||
res := Object{
|
res := Object{
|
||||||
|
|
|
@ -17,6 +17,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
|
"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -419,7 +420,7 @@ func containsACLHeaders(r *http.Request) bool {
|
||||||
r.Header.Get(api.AmzGrantFullControl) != "" || r.Header.Get(api.AmzGrantWrite) != ""
|
r.Header.Get(api.AmzGrantFullControl) != "" || r.Header.Get(api.AmzGrantWrite) != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) getNewEAclTable(r *http.Request, objInfo *api.ObjectInfo) (*eacl.Table, error) {
|
func (h *handler) getNewEAclTable(r *http.Request, objInfo *data.ObjectInfo) (*eacl.Table, error) {
|
||||||
var newEaclTable *eacl.Table
|
var newEaclTable *eacl.Table
|
||||||
objectACL, err := parseACLHeaders(r)
|
objectACL, err := parseACLHeaders(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/container"
|
"github.com/nspcc-dev/neofs-api-go/pkg/container"
|
||||||
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -20,19 +21,19 @@ import (
|
||||||
type (
|
type (
|
||||||
// BucketACL extends BucketInfo by eacl.Table.
|
// BucketACL extends BucketInfo by eacl.Table.
|
||||||
BucketACL struct {
|
BucketACL struct {
|
||||||
Info *api.BucketInfo
|
Info *data.BucketInfo
|
||||||
EACL *eacl.Table
|
EACL *eacl.Table
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*api.BucketInfo, error) {
|
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*data.BucketInfo, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
res *container.Container
|
res *container.Container
|
||||||
rid = api.GetRequestID(ctx)
|
rid = api.GetRequestID(ctx)
|
||||||
bearerOpt = n.BearerOpt(ctx)
|
bearerOpt = n.BearerOpt(ctx)
|
||||||
|
|
||||||
info = &api.BucketInfo{
|
info = &data.BucketInfo{
|
||||||
CID: cid,
|
CID: cid,
|
||||||
Name: cid.String(),
|
Name: cid.String(),
|
||||||
}
|
}
|
||||||
|
@ -83,7 +84,7 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*api.BucketInfo
|
||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) containerList(ctx context.Context) ([]*api.BucketInfo, error) {
|
func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
own = n.Owner(ctx)
|
own = n.Owner(ctx)
|
||||||
|
@ -99,7 +100,7 @@ func (n *layer) containerList(ctx context.Context) ([]*api.BucketInfo, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
list := make([]*api.BucketInfo, 0, len(res))
|
list := make([]*data.BucketInfo, 0, len(res))
|
||||||
for _, cid := range res {
|
for _, cid := range res {
|
||||||
info, err := n.containerInfo(ctx, cid)
|
info, err := n.containerInfo(ctx, cid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -117,7 +118,7 @@ func (n *layer) containerList(ctx context.Context) ([]*api.BucketInfo, error) {
|
||||||
|
|
||||||
func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*cid.ID, error) {
|
func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*cid.ID, error) {
|
||||||
var err error
|
var err error
|
||||||
bktInfo := &api.BucketInfo{
|
bktInfo := &data.BucketInfo{
|
||||||
Name: p.Name,
|
Name: p.Name,
|
||||||
Owner: n.Owner(ctx),
|
Owner: n.Owner(ctx),
|
||||||
Created: time.Now(),
|
Created: time.Now(),
|
||||||
|
|
|
@ -19,6 +19,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||||
|
@ -29,19 +30,20 @@ type (
|
||||||
layer struct {
|
layer struct {
|
||||||
pool pool.Pool
|
pool pool.Pool
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
listsCache cache.ObjectsListCache
|
listsCache *cache.ObjectsListCache
|
||||||
objCache cache.ObjectsCache
|
objCache *cache.ObjectsCache
|
||||||
namesCache cache.ObjectsNameCache
|
namesCache *cache.ObjectsNameCache
|
||||||
bucketCache cache.BucketCache
|
bucketCache *cache.BucketCache
|
||||||
systemCache cache.SystemCache
|
systemCache *cache.SystemCache
|
||||||
}
|
}
|
||||||
|
|
||||||
// CacheConfig contains params for caches.
|
// CachesConfig contains params for caches.
|
||||||
CacheConfig struct {
|
CachesConfig struct {
|
||||||
Lifetime time.Duration
|
Objects *cache.Config
|
||||||
Size int
|
ObjectsList *cache.Config
|
||||||
ListObjectsLifetime time.Duration
|
Names *cache.Config
|
||||||
ListObjectsSize int
|
Buckets *cache.Config
|
||||||
|
System *cache.Config
|
||||||
}
|
}
|
||||||
|
|
||||||
// Params stores basic API parameters.
|
// Params stores basic API parameters.
|
||||||
|
@ -55,7 +57,7 @@ type (
|
||||||
// GetObjectParams stores object get request parameters.
|
// GetObjectParams stores object get request parameters.
|
||||||
GetObjectParams struct {
|
GetObjectParams struct {
|
||||||
Range *RangeParams
|
Range *RangeParams
|
||||||
ObjectInfo *api.ObjectInfo
|
ObjectInfo *data.ObjectInfo
|
||||||
Offset int64
|
Offset int64
|
||||||
Length int64
|
Length int64
|
||||||
Writer io.Writer
|
Writer io.Writer
|
||||||
|
@ -97,7 +99,7 @@ type (
|
||||||
|
|
||||||
// CopyObjectParams stores object copy request parameters.
|
// CopyObjectParams stores object copy request parameters.
|
||||||
CopyObjectParams struct {
|
CopyObjectParams struct {
|
||||||
SrcObject *api.ObjectInfo
|
SrcObject *data.ObjectInfo
|
||||||
DstBucket string
|
DstBucket string
|
||||||
DstObject string
|
DstObject string
|
||||||
SrcSize int64
|
SrcSize int64
|
||||||
|
@ -141,7 +143,7 @@ type (
|
||||||
|
|
||||||
// PutTaggingParams stores tag set params.
|
// PutTaggingParams stores tag set params.
|
||||||
PutTaggingParams struct {
|
PutTaggingParams struct {
|
||||||
ObjectInfo *api.ObjectInfo
|
ObjectInfo *data.ObjectInfo
|
||||||
TagSet map[string]string
|
TagSet map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -154,33 +156,33 @@ type (
|
||||||
Client interface {
|
Client interface {
|
||||||
NeoFS
|
NeoFS
|
||||||
|
|
||||||
PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*api.ObjectInfo, error)
|
PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*data.ObjectInfo, error)
|
||||||
GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error)
|
GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error)
|
||||||
|
|
||||||
ListBuckets(ctx context.Context) ([]*api.BucketInfo, error)
|
ListBuckets(ctx context.Context) ([]*data.BucketInfo, error)
|
||||||
GetBucketInfo(ctx context.Context, name string) (*api.BucketInfo, error)
|
GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error)
|
||||||
GetBucketACL(ctx context.Context, name string) (*BucketACL, error)
|
GetBucketACL(ctx context.Context, name string) (*BucketACL, error)
|
||||||
PutBucketACL(ctx context.Context, p *PutBucketACLParams) error
|
PutBucketACL(ctx context.Context, p *PutBucketACLParams) error
|
||||||
CreateBucket(ctx context.Context, p *CreateBucketParams) (*cid.ID, error)
|
CreateBucket(ctx context.Context, p *CreateBucketParams) (*cid.ID, error)
|
||||||
DeleteBucket(ctx context.Context, p *DeleteBucketParams) error
|
DeleteBucket(ctx context.Context, p *DeleteBucketParams) error
|
||||||
|
|
||||||
GetObject(ctx context.Context, p *GetObjectParams) error
|
GetObject(ctx context.Context, p *GetObjectParams) error
|
||||||
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*api.ObjectInfo, error)
|
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error)
|
||||||
GetObjectTagging(ctx context.Context, p *api.ObjectInfo) (map[string]string, error)
|
GetObjectTagging(ctx context.Context, p *data.ObjectInfo) (map[string]string, error)
|
||||||
GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error)
|
GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error)
|
||||||
|
|
||||||
PutObject(ctx context.Context, p *PutObjectParams) (*api.ObjectInfo, error)
|
PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error)
|
||||||
PutObjectTagging(ctx context.Context, p *PutTaggingParams) error
|
PutObjectTagging(ctx context.Context, p *PutTaggingParams) error
|
||||||
PutBucketTagging(ctx context.Context, bucket string, tagSet map[string]string) error
|
PutBucketTagging(ctx context.Context, bucket string, tagSet map[string]string) error
|
||||||
|
|
||||||
CopyObject(ctx context.Context, p *CopyObjectParams) (*api.ObjectInfo, error)
|
CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ObjectInfo, error)
|
||||||
|
|
||||||
ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error)
|
ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error)
|
||||||
ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error)
|
ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error)
|
||||||
ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
|
ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
|
||||||
|
|
||||||
DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) ([]*VersionedObject, error)
|
DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) ([]*VersionedObject, error)
|
||||||
DeleteObjectTagging(ctx context.Context, p *api.ObjectInfo) error
|
DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) error
|
||||||
DeleteBucketTagging(ctx context.Context, bucket string) error
|
DeleteBucketTagging(ctx context.Context, bucket string) error
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
@ -194,18 +196,28 @@ func (t *VersionedObject) String() string {
|
||||||
return t.Name + ":" + t.VersionID
|
return t.Name + ":" + t.VersionID
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DefaultCachesConfigs returns filled configs.
|
||||||
|
func DefaultCachesConfigs() *CachesConfig {
|
||||||
|
return &CachesConfig{
|
||||||
|
Objects: cache.DefaultObjectsConfig(),
|
||||||
|
ObjectsList: cache.DefaultObjectsListConfig(),
|
||||||
|
Names: cache.DefaultObjectsNameConfig(),
|
||||||
|
Buckets: cache.DefaultBucketConfig(),
|
||||||
|
System: cache.DefaultSystemConfig(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// NewLayer creates instance of layer. It checks credentials
|
// NewLayer creates instance of layer. It checks credentials
|
||||||
// and establishes gRPC connection with node.
|
// and establishes gRPC connection with node.
|
||||||
func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
|
func NewLayer(log *zap.Logger, conns pool.Pool, config *CachesConfig) Client {
|
||||||
return &layer{
|
return &layer{
|
||||||
pool: conns,
|
pool: conns,
|
||||||
log: log,
|
log: log,
|
||||||
listsCache: cache.NewObjectsListCache(config.ListObjectsSize, config.ListObjectsLifetime),
|
listsCache: cache.NewObjectsListCache(config.ObjectsList),
|
||||||
objCache: cache.New(config.Size, config.Lifetime),
|
objCache: cache.New(config.Objects),
|
||||||
//todo reconsider cache params
|
namesCache: cache.NewObjectsNameCache(config.Names),
|
||||||
namesCache: cache.NewObjectsNameCache(1000, time.Minute),
|
bucketCache: cache.NewBucketCache(config.Buckets),
|
||||||
bucketCache: cache.NewBucketCache(150, time.Minute),
|
systemCache: cache.NewSystemCache(config.System),
|
||||||
systemCache: cache.NewSystemCache(1000, 5*time.Minute),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -243,7 +255,7 @@ func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Objec
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBucketInfo returns bucket info by name.
|
// GetBucketInfo returns bucket info by name.
|
||||||
func (n *layer) GetBucketInfo(ctx context.Context, name string) (*api.BucketInfo, error) {
|
func (n *layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error) {
|
||||||
name, err := url.QueryUnescape(name)
|
name, err := url.QueryUnescape(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -301,7 +313,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err
|
||||||
|
|
||||||
// ListBuckets returns all user containers. Name of the bucket is a container
|
// ListBuckets returns all user containers. Name of the bucket is a container
|
||||||
// id. Timestamp is omitted since it is not saved in neofs container.
|
// id. Timestamp is omitted since it is not saved in neofs container.
|
||||||
func (n *layer) ListBuckets(ctx context.Context) ([]*api.BucketInfo, error) {
|
func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
return n.containerList(ctx)
|
return n.containerList(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -337,7 +349,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetObjectInfo returns meta information about the object.
|
// GetObjectInfo returns meta information about the object.
|
||||||
func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*api.ObjectInfo, error) {
|
func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
||||||
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
|
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.log.Error("could not fetch bucket info", zap.Error(err))
|
n.log.Error("could not fetch bucket info", zap.Error(err))
|
||||||
|
@ -352,7 +364,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*api.Ob
|
||||||
}
|
}
|
||||||
|
|
||||||
// PutObject into storage.
|
// PutObject into storage.
|
||||||
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*api.ObjectInfo, error) {
|
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) {
|
||||||
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
|
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -362,8 +374,8 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*api.ObjectI
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetObjectTagging from storage.
|
// GetObjectTagging from storage.
|
||||||
func (n *layer) GetObjectTagging(ctx context.Context, oi *api.ObjectInfo) (map[string]string, error) {
|
func (n *layer) GetObjectTagging(ctx context.Context, oi *data.ObjectInfo) (map[string]string, error) {
|
||||||
bktInfo := &api.BucketInfo{
|
bktInfo := &data.BucketInfo{
|
||||||
Name: oi.Bucket,
|
Name: oi.Bucket,
|
||||||
CID: oi.CID,
|
CID: oi.CID,
|
||||||
Owner: oi.Owner,
|
Owner: oi.Owner,
|
||||||
|
@ -392,7 +404,7 @@ func (n *layer) GetBucketTagging(ctx context.Context, bucketName string) (map[st
|
||||||
return formTagSet(objInfo), nil
|
return formTagSet(objInfo), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func formTagSet(objInfo *api.ObjectInfo) map[string]string {
|
func formTagSet(objInfo *data.ObjectInfo) map[string]string {
|
||||||
var tagSet map[string]string
|
var tagSet map[string]string
|
||||||
if objInfo != nil {
|
if objInfo != nil {
|
||||||
tagSet = make(map[string]string, len(objInfo.Headers))
|
tagSet = make(map[string]string, len(objInfo.Headers))
|
||||||
|
@ -410,7 +422,7 @@ func formTagSet(objInfo *api.ObjectInfo) map[string]string {
|
||||||
|
|
||||||
// PutObjectTagging into storage.
|
// PutObjectTagging into storage.
|
||||||
func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error {
|
func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error {
|
||||||
bktInfo := &api.BucketInfo{
|
bktInfo := &data.BucketInfo{
|
||||||
Name: p.ObjectInfo.Bucket,
|
Name: p.ObjectInfo.Bucket,
|
||||||
CID: p.ObjectInfo.CID,
|
CID: p.ObjectInfo.CID,
|
||||||
Owner: p.ObjectInfo.Owner,
|
Owner: p.ObjectInfo.Owner,
|
||||||
|
@ -438,7 +450,7 @@ func (n *layer) PutBucketTagging(ctx context.Context, bucketName string, tagSet
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteObjectTagging from storage.
|
// DeleteObjectTagging from storage.
|
||||||
func (n *layer) DeleteObjectTagging(ctx context.Context, p *api.ObjectInfo) error {
|
func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) error {
|
||||||
bktInfo, err := n.GetBucketInfo(ctx, p.Bucket)
|
bktInfo, err := n.GetBucketInfo(ctx, p.Bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -446,7 +458,7 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *api.ObjectInfo) erro
|
||||||
return n.deleteSystemObject(ctx, bktInfo, p.TagsObject())
|
return n.deleteSystemObject(ctx, bktInfo, p.TagsObject())
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *api.BucketInfo, name string) error {
|
func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error {
|
||||||
var oid *object.ID
|
var oid *object.ID
|
||||||
if meta := n.systemCache.Get(bktInfo.SystemObjectKey(name)); meta != nil {
|
if meta := n.systemCache.Get(bktInfo.SystemObjectKey(name)); meta != nil {
|
||||||
oid = meta.ID()
|
oid = meta.ID()
|
||||||
|
@ -475,7 +487,7 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, bucketName string) erro
|
||||||
return n.deleteSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName))
|
return n.deleteSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) putSystemObject(ctx context.Context, bktInfo *api.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.Object, error) {
|
func (n *layer) putSystemObject(ctx context.Context, bktInfo *data.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.Object, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
oldOID *object.ID
|
oldOID *object.ID
|
||||||
|
@ -542,7 +554,7 @@ func (n *layer) putSystemObject(ctx context.Context, bktInfo *api.BucketInfo, ob
|
||||||
return meta, nil
|
return meta, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) getSystemObject(ctx context.Context, bkt *api.BucketInfo, objName string) (*api.ObjectInfo, error) {
|
func (n *layer) getSystemObject(ctx context.Context, bkt *data.BucketInfo, objName string) (*data.ObjectInfo, error) {
|
||||||
if meta := n.systemCache.Get(bkt.SystemObjectKey(objName)); meta != nil {
|
if meta := n.systemCache.Get(bkt.SystemObjectKey(objName)); meta != nil {
|
||||||
return objInfoFromMeta(bkt, meta), nil
|
return objInfoFromMeta(bkt, meta), nil
|
||||||
}
|
}
|
||||||
|
@ -564,7 +576,7 @@ func (n *layer) getSystemObject(ctx context.Context, bkt *api.BucketInfo, objNam
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyObject from one bucket into another bucket.
|
// CopyObject from one bucket into another bucket.
|
||||||
func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*api.ObjectInfo, error) {
|
func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ObjectInfo, error) {
|
||||||
pr, pw := io.Pipe()
|
pr, pw := io.Pipe()
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -588,7 +600,7 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*api.Objec
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteObject removes all objects with passed nice name.
|
// DeleteObject removes all objects with passed nice name.
|
||||||
func (n *layer) deleteObject(ctx context.Context, bkt *api.BucketInfo, obj *VersionedObject) *VersionedObject {
|
func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) *VersionedObject {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
ids []*object.ID
|
ids []*object.ID
|
||||||
|
@ -642,7 +654,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *api.BucketInfo, obj *Vers
|
||||||
obj.Error = err
|
obj.Error = err
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
if err = n.DeleteObjectTagging(ctx, &api.ObjectInfo{ID: id, Bucket: bkt.Name, Name: obj.Name}); err != nil {
|
if err = n.DeleteObjectTagging(ctx, &data.ObjectInfo{ID: id, Bucket: bkt.Name, Name: obj.Name}); err != nil {
|
||||||
obj.Error = err
|
obj.Error = err
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,6 +16,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -62,7 +63,7 @@ type (
|
||||||
}
|
}
|
||||||
|
|
||||||
allObjectParams struct {
|
allObjectParams struct {
|
||||||
Bucket *api.BucketInfo
|
Bucket *data.BucketInfo
|
||||||
Delimiter string
|
Delimiter string
|
||||||
Prefix string
|
Prefix string
|
||||||
}
|
}
|
||||||
|
@ -134,7 +135,7 @@ func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectPut into NeoFS, took payload from io.Reader.
|
// objectPut into NeoFS, took payload from io.Reader.
|
||||||
func (n *layer) objectPut(ctx context.Context, bkt *api.BucketInfo, p *PutObjectParams) (*api.ObjectInfo, error) {
|
func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjectParams) (*data.ObjectInfo, error) {
|
||||||
own := n.Owner(ctx)
|
own := n.Owner(ctx)
|
||||||
obj, err := url.QueryUnescape(p.Object)
|
obj, err := url.QueryUnescape(p.Object)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -198,7 +199,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *api.BucketInfo, p *PutObject
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return &api.ObjectInfo{
|
return &data.ObjectInfo{
|
||||||
ID: oid,
|
ID: oid,
|
||||||
CID: bkt.CID,
|
CID: bkt.CID,
|
||||||
|
|
||||||
|
@ -287,7 +288,7 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio
|
||||||
return idsToDeleteArr
|
return idsToDeleteArr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *api.BucketInfo, objectName string) (*api.ObjectInfo, error) {
|
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ObjectInfo, error) {
|
||||||
if address := n.namesCache.Get(bkt.Name + "/" + objectName); address != nil {
|
if address := n.namesCache.Get(bkt.Name + "/" + objectName); address != nil {
|
||||||
if headInfo := n.objCache.Get(address); headInfo != nil {
|
if headInfo := n.objCache.Get(address); headInfo != nil {
|
||||||
return objInfoFromMeta(bkt, headInfo), nil
|
return objInfoFromMeta(bkt, headInfo), nil
|
||||||
|
@ -313,7 +314,7 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *api.Bucket
|
||||||
return lastVersion, nil
|
return lastVersion, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) headVersions(ctx context.Context, bkt *api.BucketInfo, objectName string) (*objectVersions, error) {
|
func (n *layer) headVersions(ctx context.Context, bkt *data.BucketInfo, objectName string) (*objectVersions, error) {
|
||||||
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName})
|
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -351,7 +352,7 @@ func (n *layer) headVersions(ctx context.Context, bkt *api.BucketInfo, objectNam
|
||||||
return versions, nil
|
return versions, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) headVersion(ctx context.Context, bkt *api.BucketInfo, versionID string) (*api.ObjectInfo, error) {
|
func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, versionID string) (*data.ObjectInfo, error) {
|
||||||
oid := object.NewID()
|
oid := object.NewID()
|
||||||
if err := oid.Parse(versionID); err != nil {
|
if err := oid.Parse(versionID); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -396,7 +397,7 @@ func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*Lis
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
result ListObjectsInfoV1
|
result ListObjectsInfoV1
|
||||||
allObjects []*api.ObjectInfo
|
allObjects []*data.ObjectInfo
|
||||||
)
|
)
|
||||||
|
|
||||||
if p.MaxKeys == 0 {
|
if p.MaxKeys == 0 {
|
||||||
|
@ -431,7 +432,7 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
result ListObjectsInfoV2
|
result ListObjectsInfoV2
|
||||||
allObjects []*api.ObjectInfo
|
allObjects []*data.ObjectInfo
|
||||||
)
|
)
|
||||||
|
|
||||||
if p.MaxKeys == 0 {
|
if p.MaxKeys == 0 {
|
||||||
|
@ -465,13 +466,13 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
|
||||||
return &result, nil
|
return &result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*api.ObjectInfo, error) {
|
func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*data.ObjectInfo, error) {
|
||||||
versions, err := n.getAllObjectsVersions(ctx, p.Bucket, p.Prefix, p.Delimiter)
|
versions, err := n.getAllObjectsVersions(ctx, p.Bucket, p.Prefix, p.Delimiter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
objects := make([]*api.ObjectInfo, 0, len(versions))
|
objects := make([]*data.ObjectInfo, 0, len(versions))
|
||||||
for _, v := range versions {
|
for _, v := range versions {
|
||||||
lastVersion := v.getLast()
|
lastVersion := v.getLast()
|
||||||
if lastVersion != nil {
|
if lastVersion != nil {
|
||||||
|
@ -486,7 +487,7 @@ func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*ap
|
||||||
return objects, nil
|
return objects, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *api.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) {
|
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
cacheKey := cache.CreateObjectsListCacheKey(bkt.CID, prefix)
|
cacheKey := cache.CreateObjectsListCacheKey(bkt.CID, prefix)
|
||||||
|
@ -534,12 +535,12 @@ func splitVersions(header string) []string {
|
||||||
return strings.Split(header, ",")
|
return strings.Split(header, ",")
|
||||||
}
|
}
|
||||||
|
|
||||||
func isSystem(obj *api.ObjectInfo) bool {
|
func isSystem(obj *data.ObjectInfo) bool {
|
||||||
return len(obj.Headers[objectSystemAttributeName]) > 0 ||
|
return len(obj.Headers[objectSystemAttributeName]) > 0 ||
|
||||||
len(obj.Headers[attrVersionsIgnore]) > 0
|
len(obj.Headers[attrVersionsIgnore]) > 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func trimAfterObjectName(startAfter string, objects []*api.ObjectInfo) []*api.ObjectInfo {
|
func trimAfterObjectName(startAfter string, objects []*data.ObjectInfo) []*data.ObjectInfo {
|
||||||
if len(objects) != 0 && objects[len(objects)-1].Name <= startAfter {
|
if len(objects) != 0 && objects[len(objects)-1].Name <= startAfter {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -552,9 +553,9 @@ func trimAfterObjectName(startAfter string, objects []*api.ObjectInfo) []*api.Ob
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func trimAfterObjectID(id string, objects []*api.ObjectInfo) []*api.ObjectInfo {
|
func trimAfterObjectID(id string, objects []*data.ObjectInfo) []*data.ObjectInfo {
|
||||||
if len(objects) != 0 && objects[len(objects)-1].ID.String() == id {
|
if len(objects) != 0 && objects[len(objects)-1].ID.String() == id {
|
||||||
return []*api.ObjectInfo{}
|
return []*data.ObjectInfo{}
|
||||||
}
|
}
|
||||||
for i, obj := range objects {
|
for i, obj := range objects {
|
||||||
if obj.ID.String() == id {
|
if obj.ID.String() == id {
|
||||||
|
@ -565,7 +566,7 @@ func trimAfterObjectID(id string, objects []*api.ObjectInfo) []*api.ObjectInfo {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func triageObjects(allObjects []*api.ObjectInfo) (prefixes []string, objects []*api.ObjectInfo) {
|
func triageObjects(allObjects []*data.ObjectInfo) (prefixes []string, objects []*data.ObjectInfo) {
|
||||||
for _, ov := range allObjects {
|
for _, ov := range allObjects {
|
||||||
if ov.IsDir {
|
if ov.IsDir {
|
||||||
prefixes = append(prefixes, ov.Name)
|
prefixes = append(prefixes, ov.Name)
|
||||||
|
@ -577,11 +578,11 @@ func triageObjects(allObjects []*api.ObjectInfo) (prefixes []string, objects []*
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*api.ObjectInfo, error) {
|
func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*data.ObjectInfo, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
bkt *api.BucketInfo
|
bkt *data.BucketInfo
|
||||||
allObjects []*api.ObjectInfo
|
allObjects []*data.ObjectInfo
|
||||||
)
|
)
|
||||||
|
|
||||||
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
|
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
|
||||||
|
@ -600,7 +601,7 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
|
||||||
return allObjects, nil
|
return allObjects, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *api.BucketInfo) bool {
|
func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *data.BucketInfo) bool {
|
||||||
settings, err := n.getBucketSettings(ctx, bktInfo)
|
settings, err := n.getBucketSettings(ctx, bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.log.Warn("couldn't get versioning settings object", zap.Error(err))
|
n.log.Warn("couldn't get versioning settings object", zap.Error(err))
|
||||||
|
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -26,11 +26,11 @@ func randSHA256Checksum(t *testing.T) (cs [sha256.Size]byte) {
|
||||||
|
|
||||||
func TestTrimAfterObjectName(t *testing.T) {
|
func TestTrimAfterObjectName(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
objects []*api.ObjectInfo
|
objects []*data.ObjectInfo
|
||||||
names = []string{"b", "c", "d"}
|
names = []string{"b", "c", "d"}
|
||||||
)
|
)
|
||||||
for _, name := range names {
|
for _, name := range names {
|
||||||
objects = append(objects, &api.ObjectInfo{Name: name})
|
objects = append(objects, &data.ObjectInfo{Name: name})
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("startafter before all objects", func(t *testing.T) {
|
t.Run("startafter before all objects", func(t *testing.T) {
|
||||||
|
@ -59,7 +59,7 @@ func TestTrimAfterObjectName(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("empty objects", func(t *testing.T) {
|
t.Run("empty objects", func(t *testing.T) {
|
||||||
actual := trimAfterObjectName(names[0], []*api.ObjectInfo{})
|
actual := trimAfterObjectName(names[0], []*data.ObjectInfo{})
|
||||||
require.Nil(t, actual)
|
require.Nil(t, actual)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -76,14 +76,14 @@ func TestTrimAfterObjectName(t *testing.T) {
|
||||||
|
|
||||||
func TestTrimAfterObjectID(t *testing.T) {
|
func TestTrimAfterObjectID(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
objects []*api.ObjectInfo
|
objects []*data.ObjectInfo
|
||||||
ids []*object.ID
|
ids []*object.ID
|
||||||
numberOfIDS = 3
|
numberOfIDS = 3
|
||||||
)
|
)
|
||||||
|
|
||||||
for i := 0; i < numberOfIDS; i++ {
|
for i := 0; i < numberOfIDS; i++ {
|
||||||
id := randID(t)
|
id := randID(t)
|
||||||
objects = append(objects, &api.ObjectInfo{ID: id})
|
objects = append(objects, &data.ObjectInfo{ID: id})
|
||||||
ids = append(ids, id)
|
ids = append(ids, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -17,7 +18,7 @@ type (
|
||||||
// ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2.
|
// ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2.
|
||||||
ListObjectsInfo struct {
|
ListObjectsInfo struct {
|
||||||
Prefixes []string
|
Prefixes []string
|
||||||
Objects []*api.ObjectInfo
|
Objects []*data.ObjectInfo
|
||||||
IsTruncated bool
|
IsTruncated bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -35,7 +36,7 @@ type (
|
||||||
|
|
||||||
// ObjectVersionInfo stores info about objects versions.
|
// ObjectVersionInfo stores info about objects versions.
|
||||||
ObjectVersionInfo struct {
|
ObjectVersionInfo struct {
|
||||||
Object *api.ObjectInfo
|
Object *data.ObjectInfo
|
||||||
IsLatest bool
|
IsLatest bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -65,11 +66,11 @@ func userHeaders(attrs []*object.Attribute) map[string]string {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func objInfoFromMeta(bkt *api.BucketInfo, meta *object.Object) *api.ObjectInfo {
|
func objInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectInfo {
|
||||||
return objectInfoFromMeta(bkt, meta, "", "")
|
return objectInfoFromMeta(bkt, meta, "", "")
|
||||||
}
|
}
|
||||||
|
|
||||||
func objectInfoFromMeta(bkt *api.BucketInfo, meta *object.Object, prefix, delimiter string) *api.ObjectInfo {
|
func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object, prefix, delimiter string) *data.ObjectInfo {
|
||||||
var (
|
var (
|
||||||
isDir bool
|
isDir bool
|
||||||
size int64
|
size int64
|
||||||
|
@ -110,7 +111,7 @@ func objectInfoFromMeta(bkt *api.BucketInfo, meta *object.Object, prefix, delimi
|
||||||
size = int64(meta.PayloadSize())
|
size = int64(meta.PayloadSize())
|
||||||
}
|
}
|
||||||
|
|
||||||
return &api.ObjectInfo{
|
return &data.ObjectInfo{
|
||||||
ID: meta.ID(),
|
ID: meta.ID(),
|
||||||
CID: bkt.CID,
|
CID: bkt.CID,
|
||||||
IsDir: isDir,
|
IsDir: isDir,
|
||||||
|
|
|
@ -9,7 +9,7 @@ import (
|
||||||
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -20,7 +20,7 @@ var (
|
||||||
defaultTestContentType = http.DetectContentType(defaultTestPayload)
|
defaultTestContentType = http.DetectContentType(defaultTestPayload)
|
||||||
)
|
)
|
||||||
|
|
||||||
func newTestObject(oid *object.ID, bkt *api.BucketInfo, name string) *object.Object {
|
func newTestObject(oid *object.ID, bkt *data.BucketInfo, name string) *object.Object {
|
||||||
filename := object.NewAttribute()
|
filename := object.NewAttribute()
|
||||||
filename.SetKey(object.AttributeFileName)
|
filename.SetKey(object.AttributeFileName)
|
||||||
filename.SetValue(name)
|
filename.SetValue(name)
|
||||||
|
@ -44,8 +44,8 @@ func newTestObject(oid *object.ID, bkt *api.BucketInfo, name string) *object.Obj
|
||||||
return raw.Object()
|
return raw.Object()
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTestInfo(oid *object.ID, bkt *api.BucketInfo, name string, isDir bool) *api.ObjectInfo {
|
func newTestInfo(oid *object.ID, bkt *data.BucketInfo, name string, isDir bool) *data.ObjectInfo {
|
||||||
info := &api.ObjectInfo{
|
info := &data.ObjectInfo{
|
||||||
ID: oid,
|
ID: oid,
|
||||||
Name: name,
|
Name: name,
|
||||||
Bucket: bkt.Name,
|
Bucket: bkt.Name,
|
||||||
|
@ -72,7 +72,7 @@ func Test_objectInfoFromMeta(t *testing.T) {
|
||||||
oid := object.NewID()
|
oid := object.NewID()
|
||||||
containerID := cid.New()
|
containerID := cid.New()
|
||||||
|
|
||||||
bkt := &api.BucketInfo{
|
bkt := &data.BucketInfo{
|
||||||
Name: "test-container",
|
Name: "test-container",
|
||||||
CID: containerID,
|
CID: containerID,
|
||||||
Owner: uid,
|
Owner: uid,
|
||||||
|
@ -82,7 +82,7 @@ func Test_objectInfoFromMeta(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
prefix string
|
prefix string
|
||||||
result *api.ObjectInfo
|
result *data.ObjectInfo
|
||||||
object *object.Object
|
object *object.Object
|
||||||
delimiter string
|
delimiter string
|
||||||
}{
|
}{
|
||||||
|
|
|
@ -8,13 +8,13 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
type objectVersions struct {
|
type objectVersions struct {
|
||||||
name string
|
name string
|
||||||
objects []*api.ObjectInfo
|
objects []*data.ObjectInfo
|
||||||
addList []string
|
addList []string
|
||||||
delList []string
|
delList []string
|
||||||
isSorted bool
|
isSorted bool
|
||||||
|
@ -40,7 +40,7 @@ func (v *objectVersions) isAddListEmpty() bool {
|
||||||
return len(v.addList) == 0
|
return len(v.addList) == 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *objectVersions) appendVersion(oi *api.ObjectInfo) {
|
func (v *objectVersions) appendVersion(oi *data.ObjectInfo) {
|
||||||
delVers := splitVersions(oi.Headers[versionsDelAttr])
|
delVers := splitVersions(oi.Headers[versionsDelAttr])
|
||||||
v.objects = append(v.objects, oi)
|
v.objects = append(v.objects, oi)
|
||||||
|
|
||||||
|
@ -100,7 +100,7 @@ func (v *objectVersions) formAddList() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func containsVersions(obj *api.ObjectInfo, versions []string) bool {
|
func containsVersions(obj *data.ObjectInfo, versions []string) bool {
|
||||||
header := obj.Headers[versionsAddAttr]
|
header := obj.Headers[versionsAddAttr]
|
||||||
for _, version := range versions {
|
for _, version := range versions {
|
||||||
if !strings.Contains(header, version) {
|
if !strings.Contains(header, version) {
|
||||||
|
@ -156,7 +156,7 @@ LOOP:
|
||||||
return commonAddedVersions, prevVersions, currentVersions
|
return commonAddedVersions, prevVersions, currentVersions
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *objectVersions) getLast() *api.ObjectInfo {
|
func (v *objectVersions) getLast() *data.ObjectInfo {
|
||||||
if v == nil || len(v.objects) == 0 {
|
if v == nil || len(v.objects) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -189,14 +189,14 @@ func (v *objectVersions) existedVersions() []string {
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *objectVersions) getFiltered(reverse bool) []*api.ObjectInfo {
|
func (v *objectVersions) getFiltered(reverse bool) []*data.ObjectInfo {
|
||||||
if len(v.objects) == 0 {
|
if len(v.objects) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
v.sort()
|
v.sort()
|
||||||
existedVersions := v.existedVersions()
|
existedVersions := v.existedVersions()
|
||||||
res := make([]*api.ObjectInfo, 0, len(v.objects))
|
res := make([]*data.ObjectInfo, 0, len(v.objects))
|
||||||
|
|
||||||
for _, version := range v.objects {
|
for _, version := range v.objects {
|
||||||
delMark := version.Headers[versionsDeleteMarkAttr]
|
delMark := version.Headers[versionsDeleteMarkAttr]
|
||||||
|
@ -223,7 +223,7 @@ func (v *objectVersions) getDelHeader() string {
|
||||||
return strings.Join(v.delList, ",")
|
return strings.Join(v.delList, ",")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *objectVersions) getVersion(oid *object.ID) *api.ObjectInfo {
|
func (v *objectVersions) getVersion(oid *object.ID) *data.ObjectInfo {
|
||||||
for _, version := range v.objects {
|
for _, version := range v.objects {
|
||||||
if version.Version() == oid.String() {
|
if version.Version() == oid.String() {
|
||||||
if contains(v.delList, oid.String()) {
|
if contains(v.delList, oid.String()) {
|
||||||
|
@ -234,7 +234,7 @@ func (v *objectVersions) getVersion(oid *object.ID) *api.ObjectInfo {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*api.ObjectInfo, error) {
|
func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*data.ObjectInfo, error) {
|
||||||
bktInfo, err := n.GetBucketInfo(ctx, p.Bucket)
|
bktInfo, err := n.GetBucketInfo(ctx, p.Bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -264,7 +264,7 @@ func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*Bu
|
||||||
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||||
var (
|
var (
|
||||||
versions map[string]*objectVersions
|
versions map[string]*objectVersions
|
||||||
allObjects = make([]*api.ObjectInfo, 0, p.MaxKeys)
|
allObjects = make([]*data.ObjectInfo, 0, p.MaxKeys)
|
||||||
res = &ListObjectVersionsInfo{}
|
res = &ListObjectVersionsInfo{}
|
||||||
reverse = true
|
reverse = true
|
||||||
)
|
)
|
||||||
|
@ -347,7 +347,7 @@ func contains(list []string, elem string) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) getBucketSettings(ctx context.Context, bktInfo *api.BucketInfo) (*BucketSettings, error) {
|
func (n *layer) getBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*BucketSettings, error) {
|
||||||
objInfo, err := n.getSystemObject(ctx, bktInfo, bktInfo.SettingsObjectName())
|
objInfo, err := n.getSystemObject(ctx, bktInfo, bktInfo.SettingsObjectName())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -356,7 +356,7 @@ func (n *layer) getBucketSettings(ctx context.Context, bktInfo *api.BucketInfo)
|
||||||
return objectInfoToBucketSettings(objInfo), nil
|
return objectInfoToBucketSettings(objInfo), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func objectInfoToBucketSettings(info *api.ObjectInfo) *BucketSettings {
|
func objectInfoToBucketSettings(info *data.ObjectInfo) *BucketSettings {
|
||||||
res := &BucketSettings{}
|
res := &BucketSettings{}
|
||||||
|
|
||||||
enabled, ok := info.Headers[attrSettingsVersioningEnabled]
|
enabled, ok := info.Headers[attrSettingsVersioningEnabled]
|
||||||
|
@ -368,7 +368,7 @@ func objectInfoToBucketSettings(info *api.ObjectInfo) *BucketSettings {
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) checkVersionsExist(ctx context.Context, bkt *api.BucketInfo, obj *VersionedObject) (*api.ObjectInfo, error) {
|
func (n *layer) checkVersionsExist(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.ObjectInfo, error) {
|
||||||
id := object.NewID()
|
id := object.NewID()
|
||||||
if err := id.Parse(obj.VersionID); err != nil {
|
if err := id.Parse(obj.VersionID); err != nil {
|
||||||
return nil, errors.GetAPIError(errors.ErrInvalidVersion)
|
return nil, errors.GetAPIError(errors.ErrInvalidVersion)
|
||||||
|
|
|
@ -21,7 +21,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/session"
|
"github.com/nspcc-dev/neofs-api-go/pkg/session"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/token"
|
"github.com/nspcc-dev/neofs-api-go/pkg/token"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/logger"
|
"github.com/nspcc-dev/neofs-sdk-go/logger"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||||
|
@ -208,7 +208,7 @@ func (t *testPool) WaitForContainerPresence(ctx context.Context, id *cid.ID, par
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tc *testContext) putObject(content []byte) *api.ObjectInfo {
|
func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
|
||||||
objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
|
objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
|
||||||
Bucket: tc.bkt,
|
Bucket: tc.bkt,
|
||||||
Object: tc.obj,
|
Object: tc.obj,
|
||||||
|
@ -221,7 +221,7 @@ func (tc *testContext) putObject(content []byte) *api.ObjectInfo {
|
||||||
return objInfo
|
return objInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tc *testContext) getObject(objectName, versionID string, needError bool) (*api.ObjectInfo, []byte) {
|
func (tc *testContext) getObject(objectName, versionID string, needError bool) (*data.ObjectInfo, []byte) {
|
||||||
objInfo, err := tc.layer.GetObjectInfo(tc.ctx, &HeadObjectParams{
|
objInfo, err := tc.layer.GetObjectInfo(tc.ctx, &HeadObjectParams{
|
||||||
Bucket: tc.bkt,
|
Bucket: tc.bkt,
|
||||||
Object: objectName,
|
Object: objectName,
|
||||||
|
@ -254,7 +254,7 @@ func (tc *testContext) deleteObject(objectName, versionID string) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tc *testContext) listObjectsV1() []*api.ObjectInfo {
|
func (tc *testContext) listObjectsV1() []*data.ObjectInfo {
|
||||||
res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{
|
res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{
|
||||||
ListObjectsParamsCommon: ListObjectsParamsCommon{
|
ListObjectsParamsCommon: ListObjectsParamsCommon{
|
||||||
Bucket: tc.bkt,
|
Bucket: tc.bkt,
|
||||||
|
@ -265,7 +265,7 @@ func (tc *testContext) listObjectsV1() []*api.ObjectInfo {
|
||||||
return res.Objects
|
return res.Objects
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tc *testContext) listObjectsV2() []*api.ObjectInfo {
|
func (tc *testContext) listObjectsV2() []*data.ObjectInfo {
|
||||||
res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{
|
res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{
|
||||||
ListObjectsParamsCommon: ListObjectsParamsCommon{
|
ListObjectsParamsCommon: ListObjectsParamsCommon{
|
||||||
Bucket: tc.bkt,
|
Bucket: tc.bkt,
|
||||||
|
@ -330,12 +330,7 @@ func prepareContext(t *testing.T) *testContext {
|
||||||
|
|
||||||
return &testContext{
|
return &testContext{
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
layer: NewLayer(l, tp, &CacheConfig{
|
layer: NewLayer(l, tp, DefaultCachesConfigs()),
|
||||||
Size: cache.DefaultObjectsCacheSize,
|
|
||||||
Lifetime: cache.DefaultObjectsCacheLifetime,
|
|
||||||
ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime,
|
|
||||||
ListObjectsSize: cache.DefaultObjectsListCacheSize},
|
|
||||||
),
|
|
||||||
bkt: bktName,
|
bkt: bktName,
|
||||||
bktID: bktID,
|
bktID: bktID,
|
||||||
obj: "obj1",
|
obj: "obj1",
|
||||||
|
@ -457,7 +452,7 @@ func TestGetLastVersion(t *testing.T) {
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
versions *objectVersions
|
versions *objectVersions
|
||||||
expected *api.ObjectInfo
|
expected *data.ObjectInfo
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
versions: &objectVersions{},
|
versions: &objectVersions{},
|
||||||
|
@ -465,21 +460,21 @@ func TestGetLastVersion(t *testing.T) {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{
|
versions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj2, obj1},
|
objects: []*data.ObjectInfo{obj2, obj1},
|
||||||
addList: []string{obj1.Version(), obj2.Version()},
|
addList: []string{obj1.Version(), obj2.Version()},
|
||||||
},
|
},
|
||||||
expected: obj2,
|
expected: obj2,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{
|
versions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj2, obj1, obj3},
|
objects: []*data.ObjectInfo{obj2, obj1, obj3},
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
|
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
|
||||||
},
|
},
|
||||||
expected: nil,
|
expected: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{
|
versions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj2, obj1, obj4},
|
objects: []*data.ObjectInfo{obj2, obj1, obj4},
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
|
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
|
||||||
delList: []string{obj2.Version()},
|
delList: []string{obj2.Version()},
|
||||||
},
|
},
|
||||||
|
@ -487,7 +482,7 @@ func TestGetLastVersion(t *testing.T) {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{
|
versions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj1, obj5},
|
objects: []*data.ObjectInfo{obj1, obj5},
|
||||||
addList: []string{obj1.Version(), obj5.Version()},
|
addList: []string{obj1.Version(), obj5.Version()},
|
||||||
delList: []string{obj1.Version()},
|
delList: []string{obj1.Version()},
|
||||||
},
|
},
|
||||||
|
@ -495,13 +490,13 @@ func TestGetLastVersion(t *testing.T) {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{
|
versions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj5},
|
objects: []*data.ObjectInfo{obj5},
|
||||||
},
|
},
|
||||||
expected: nil,
|
expected: nil,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{
|
versions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj1, obj2, obj3, obj6},
|
objects: []*data.ObjectInfo{obj1, obj2, obj3, obj6},
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj6.Version()},
|
addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj6.Version()},
|
||||||
delList: []string{obj3.Version()},
|
delList: []string{obj3.Version()},
|
||||||
},
|
},
|
||||||
|
@ -509,7 +504,7 @@ func TestGetLastVersion(t *testing.T) {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{
|
versions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj1, obj1V2},
|
objects: []*data.ObjectInfo{obj1, obj1V2},
|
||||||
addList: []string{obj1.Version(), obj1V2.Version()},
|
addList: []string{obj1.Version(), obj1V2.Version()},
|
||||||
},
|
},
|
||||||
expected: obj1V2,
|
expected: obj1V2,
|
||||||
|
@ -530,51 +525,51 @@ func TestAppendVersions(t *testing.T) {
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
versions *objectVersions
|
versions *objectVersions
|
||||||
objectToAdd *api.ObjectInfo
|
objectToAdd *data.ObjectInfo
|
||||||
expectedVersions *objectVersions
|
expectedVersions *objectVersions
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
versions: &objectVersions{},
|
versions: &objectVersions{},
|
||||||
objectToAdd: obj1,
|
objectToAdd: obj1,
|
||||||
expectedVersions: &objectVersions{
|
expectedVersions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj1},
|
objects: []*data.ObjectInfo{obj1},
|
||||||
addList: []string{obj1.Version()},
|
addList: []string{obj1.Version()},
|
||||||
isSorted: true,
|
isSorted: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{objects: []*api.ObjectInfo{obj1}},
|
versions: &objectVersions{objects: []*data.ObjectInfo{obj1}},
|
||||||
objectToAdd: obj2,
|
objectToAdd: obj2,
|
||||||
expectedVersions: &objectVersions{
|
expectedVersions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj1, obj2},
|
objects: []*data.ObjectInfo{obj1, obj2},
|
||||||
addList: []string{obj1.Version(), obj2.Version()},
|
addList: []string{obj1.Version(), obj2.Version()},
|
||||||
isSorted: true,
|
isSorted: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{objects: []*api.ObjectInfo{obj1, obj2}},
|
versions: &objectVersions{objects: []*data.ObjectInfo{obj1, obj2}},
|
||||||
objectToAdd: obj3,
|
objectToAdd: obj3,
|
||||||
expectedVersions: &objectVersions{
|
expectedVersions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj1, obj2, obj3},
|
objects: []*data.ObjectInfo{obj1, obj2, obj3},
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
|
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
|
||||||
isSorted: true,
|
isSorted: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{objects: []*api.ObjectInfo{obj1, obj2}},
|
versions: &objectVersions{objects: []*data.ObjectInfo{obj1, obj2}},
|
||||||
objectToAdd: obj4,
|
objectToAdd: obj4,
|
||||||
expectedVersions: &objectVersions{
|
expectedVersions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj1, obj2, obj4},
|
objects: []*data.ObjectInfo{obj1, obj2, obj4},
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
|
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
|
||||||
delList: []string{obj2.Version()},
|
delList: []string{obj2.Version()},
|
||||||
isSorted: true,
|
isSorted: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{objects: []*api.ObjectInfo{obj5}},
|
versions: &objectVersions{objects: []*data.ObjectInfo{obj5}},
|
||||||
objectToAdd: obj6,
|
objectToAdd: obj6,
|
||||||
expectedVersions: &objectVersions{
|
expectedVersions: &objectVersions{
|
||||||
objects: []*api.ObjectInfo{obj5, obj6},
|
objects: []*data.ObjectInfo{obj5, obj6},
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj5.Version(), obj6.Version()},
|
addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj5.Version(), obj6.Version()},
|
||||||
isSorted: true,
|
isSorted: true,
|
||||||
},
|
},
|
||||||
|
@ -606,15 +601,15 @@ func TestSortAddHeaders(t *testing.T) {
|
||||||
expectedAddHeaders string
|
expectedAddHeaders string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
versions: &objectVersions{objects: []*api.ObjectInfo{obj6, obj7, obj8}},
|
versions: &objectVersions{objects: []*data.ObjectInfo{obj6, obj7, obj8}},
|
||||||
expectedAddHeaders: joinVers(obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8),
|
expectedAddHeaders: joinVers(obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{objects: []*api.ObjectInfo{obj7, obj9}},
|
versions: &objectVersions{objects: []*data.ObjectInfo{obj7, obj9}},
|
||||||
expectedAddHeaders: joinVers(obj1, obj4, obj5, obj7, obj9),
|
expectedAddHeaders: joinVers(obj1, obj4, obj5, obj7, obj9),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
versions: &objectVersions{objects: []*api.ObjectInfo{obj11, obj10, obj12}},
|
versions: &objectVersions{objects: []*data.ObjectInfo{obj11, obj10, obj12}},
|
||||||
expectedAddHeaders: joinVers(obj10, obj11, obj12),
|
expectedAddHeaders: joinVers(obj10, obj11, obj12),
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
|
@ -622,7 +617,7 @@ func TestSortAddHeaders(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func joinVers(objs ...*api.ObjectInfo) string {
|
func joinVers(objs ...*data.ObjectInfo) string {
|
||||||
if len(objs) == 0 {
|
if len(objs) == 0 {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
@ -643,7 +638,7 @@ func getOID(id byte) *object.ID {
|
||||||
return oid
|
return oid
|
||||||
}
|
}
|
||||||
|
|
||||||
func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *api.ObjectInfo {
|
func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo {
|
||||||
headers := make(map[string]string)
|
headers := make(map[string]string)
|
||||||
if addAttr != "" {
|
if addAttr != "" {
|
||||||
headers[versionsAddAttr] = addAttr
|
headers[versionsAddAttr] = addAttr
|
||||||
|
@ -655,14 +650,14 @@ func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *api.Objec
|
||||||
headers[versionsDeleteMarkAttr] = delMarkAttr
|
headers[versionsDeleteMarkAttr] = delMarkAttr
|
||||||
}
|
}
|
||||||
|
|
||||||
return &api.ObjectInfo{
|
return &data.ObjectInfo{
|
||||||
ID: getOID(id),
|
ID: getOID(id),
|
||||||
Name: strconv.Itoa(int(id)),
|
Name: strconv.Itoa(int(id)),
|
||||||
Headers: headers,
|
Headers: headers,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getTestObjectInfoEpoch(epoch uint64, id byte, addAttr, delAttr, delMarkAttr string) *api.ObjectInfo {
|
func getTestObjectInfoEpoch(epoch uint64, id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo {
|
||||||
obj := getTestObjectInfo(id, addAttr, delAttr, delMarkAttr)
|
obj := getTestObjectInfo(id, addAttr, delAttr, delMarkAttr)
|
||||||
obj.CreationEpoch = epoch
|
obj.CreationEpoch = epoch
|
||||||
return obj
|
return obj
|
||||||
|
|
|
@ -22,6 +22,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/session"
|
"github.com/nspcc-dev/neofs-api-go/pkg/session"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/token"
|
"github.com/nspcc-dev/neofs-api-go/pkg/token"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/policy"
|
"github.com/nspcc-dev/neofs-sdk-go/policy"
|
||||||
|
@ -226,7 +227,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
||||||
}
|
}
|
||||||
|
|
||||||
address, err := tokens.
|
address, err := tokens.
|
||||||
New(a.pool, secrets.EphemeralKey, tokens.DefaultCacheConfig()).
|
New(a.pool, secrets.EphemeralKey, cache.DefaultAccessBoxConfig()).
|
||||||
Put(ctx, cid, oid, box, lifetime.Exp, options.GatesPublicKeys...)
|
Put(ctx, cid, oid, box, lifetime.Exp, options.GatesPublicKeys...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to put bearer token: %w", err)
|
return fmt.Errorf("failed to put bearer token: %w", err)
|
||||||
|
@ -268,7 +269,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
||||||
// ObtainSecret receives an existing secret access key from NeoFS and
|
// ObtainSecret receives an existing secret access key from NeoFS and
|
||||||
// writes to io.Writer the secret access key.
|
// writes to io.Writer the secret access key.
|
||||||
func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error {
|
func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error {
|
||||||
bearerCreds := tokens.New(a.pool, options.GatePrivateKey, tokens.DefaultCacheConfig())
|
bearerCreds := tokens.New(a.pool, options.GatePrivateKey, cache.DefaultAccessBoxConfig())
|
||||||
address := object.NewAddress()
|
address := object.NewAddress()
|
||||||
if err := address.Parse(options.SecretAddress); err != nil {
|
if err := address.Parse(options.SecretAddress); err != nil {
|
||||||
return fmt.Errorf("failed to parse secret address: %w", err)
|
return fmt.Errorf("failed to parse secret address: %w", err)
|
||||||
|
|
|
@ -14,7 +14,6 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
|
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
|
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/policy"
|
"github.com/nspcc-dev/neofs-sdk-go/policy"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||||
|
@ -216,21 +215,25 @@ func (a *App) Server(ctx context.Context) {
|
||||||
close(a.webDone)
|
close(a.webDone)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig {
|
func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CachesConfig {
|
||||||
cacheCfg := layer.CacheConfig{
|
cacheCfg := layer.DefaultCachesConfigs()
|
||||||
ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime,
|
|
||||||
ListObjectsSize: cache.DefaultObjectsListCacheSize,
|
|
||||||
Size: cache.DefaultObjectsCacheSize,
|
|
||||||
Lifetime: cache.DefaultObjectsCacheLifetime,
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheCfg.Lifetime = getLifetime(v, l, cfgObjectsCacheLifetime, cacheCfg.Lifetime)
|
cacheCfg.Objects.Lifetime = getLifetime(v, l, cfgObjectsCacheLifetime, cacheCfg.Objects.Lifetime)
|
||||||
cacheCfg.Size = getSize(v, l, cfgObjectsCacheSize, cacheCfg.Size)
|
cacheCfg.Objects.Size = getSize(v, l, cfgObjectsCacheSize, cacheCfg.Objects.Size)
|
||||||
|
|
||||||
cacheCfg.ListObjectsLifetime = getLifetime(v, l, cfgListObjectsCacheLifetime, cacheCfg.ListObjectsLifetime)
|
cacheCfg.ObjectsList.Lifetime = getLifetime(v, l, cfgListObjectsCacheLifetime, cacheCfg.ObjectsList.Lifetime)
|
||||||
cacheCfg.ListObjectsSize = getSize(v, l, cfgListObjectsCacheSize, cacheCfg.ListObjectsSize)
|
cacheCfg.ObjectsList.Size = getSize(v, l, cfgListObjectsCacheSize, cacheCfg.ObjectsList.Size)
|
||||||
|
|
||||||
return &cacheCfg
|
cacheCfg.Buckets.Lifetime = getLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Buckets.Lifetime)
|
||||||
|
cacheCfg.Buckets.Size = getSize(v, l, cfgBucketsCacheSize, cacheCfg.Buckets.Size)
|
||||||
|
|
||||||
|
cacheCfg.Names.Lifetime = getLifetime(v, l, cfgNamesCacheLifetime, cacheCfg.Names.Lifetime)
|
||||||
|
cacheCfg.Names.Size = getSize(v, l, cfgNamesCacheSize, cacheCfg.Names.Size)
|
||||||
|
|
||||||
|
cacheCfg.System.Lifetime = getLifetime(v, l, cfgSystemLifetimeSize, cacheCfg.System.Lifetime)
|
||||||
|
cacheCfg.System.Size = getSize(v, l, cfgSystemCacheSize, cacheCfg.System.Size)
|
||||||
|
|
||||||
|
return cacheCfg
|
||||||
}
|
}
|
||||||
|
|
||||||
func getLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration {
|
func getLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration {
|
||||||
|
@ -263,8 +266,8 @@ func getSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue int) i
|
||||||
return defaultValue
|
return defaultValue
|
||||||
}
|
}
|
||||||
|
|
||||||
func getAccessBoxCacheConfig(v *viper.Viper, l *zap.Logger) *tokens.CacheConfig {
|
func getAccessBoxCacheConfig(v *viper.Viper, l *zap.Logger) *cache.Config {
|
||||||
cacheCfg := tokens.DefaultCacheConfig()
|
cacheCfg := cache.DefaultAccessBoxConfig()
|
||||||
|
|
||||||
cacheCfg.Lifetime = getLifetime(v, l, cfgAccessBoxCacheLifetime, cacheCfg.Lifetime)
|
cacheCfg.Lifetime = getLifetime(v, l, cfgAccessBoxCacheLifetime, cacheCfg.Lifetime)
|
||||||
cacheCfg.Size = getSize(v, l, cfgAccessBoxCacheSize, cacheCfg.Size)
|
cacheCfg.Size = getSize(v, l, cfgAccessBoxCacheSize, cacheCfg.Size)
|
||||||
|
|
|
@ -59,6 +59,12 @@ const ( // Settings.
|
||||||
cfgObjectsCacheSize = "cache.objects.size"
|
cfgObjectsCacheSize = "cache.objects.size"
|
||||||
cfgListObjectsCacheLifetime = "cache.list.lifetime"
|
cfgListObjectsCacheLifetime = "cache.list.lifetime"
|
||||||
cfgListObjectsCacheSize = "cache.list.size"
|
cfgListObjectsCacheSize = "cache.list.size"
|
||||||
|
cfgBucketsCacheLifetime = "cache.buckets.lifetime"
|
||||||
|
cfgBucketsCacheSize = "cache.buckets.size"
|
||||||
|
cfgNamesCacheLifetime = "cache.names.lifetime"
|
||||||
|
cfgNamesCacheSize = "cache.names.size"
|
||||||
|
cfgSystemLifetimeSize = "cache.system.lifetime"
|
||||||
|
cfgSystemCacheSize = "cache.system.size"
|
||||||
cfgAccessBoxCacheLifetime = "cache.accessbox.lifetime"
|
cfgAccessBoxCacheLifetime = "cache.accessbox.lifetime"
|
||||||
cfgAccessBoxCacheSize = "cache.accessbox.size"
|
cfgAccessBoxCacheSize = "cache.accessbox.size"
|
||||||
|
|
||||||
|
|
|
@ -13,6 +13,7 @@ import (
|
||||||
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||||
)
|
)
|
||||||
|
@ -27,7 +28,7 @@ type (
|
||||||
cred struct {
|
cred struct {
|
||||||
key *keys.PrivateKey
|
key *keys.PrivateKey
|
||||||
pool pool.Pool
|
pool pool.Pool
|
||||||
cache *AccessBoxCache
|
cache *cache.AccessBoxCache
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -47,8 +48,8 @@ var bufferPool = sync.Pool{
|
||||||
var _ = New
|
var _ = New
|
||||||
|
|
||||||
// New creates new Credentials instance using given cli and key.
|
// New creates new Credentials instance using given cli and key.
|
||||||
func New(conns pool.Pool, key *keys.PrivateKey, config *CacheConfig) Credentials {
|
func New(conns pool.Pool, key *keys.PrivateKey, config *cache.Config) Credentials {
|
||||||
return &cred{pool: conns, key: key, cache: NewAccessBoxCache(config)}
|
return &cred{pool: conns, key: key, cache: cache.NewAccessBoxCache(config)}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *cred) acquireBuffer() *bytes.Buffer {
|
func (c *cred) acquireBuffer() *bytes.Buffer {
|
||||||
|
|
|
@ -114,6 +114,15 @@ cache:
|
||||||
list:
|
list:
|
||||||
lifetime: 1m
|
lifetime: 1m
|
||||||
size: 100
|
size: 100
|
||||||
|
names:
|
||||||
|
lifetime: 1m
|
||||||
|
size: 1000
|
||||||
|
buckets:
|
||||||
|
lifetime: 1m
|
||||||
|
size: 500
|
||||||
|
system:
|
||||||
|
lifetime: 2m
|
||||||
|
size: 1000
|
||||||
accessbox:
|
accessbox:
|
||||||
lifetime: 5m
|
lifetime: 5m
|
||||||
size: 10
|
size: 10
|
||||||
|
|
Loading…
Reference in a new issue