acid-ant/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#253] Caches refactoring

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-09-10 09:56:56 +03:00 committed by Kirillov Denis
parent 951eb6fda8
commit 19b917e3b5
30 changed files with 365 additions and 322 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
api/auth/center.go
View file

@ -18,6 +18,7 @@ import (
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
@ -81,7 +82,7 @@ func (p prs) Seek(_ int64, _ int) (int64, error) {
var _ io.ReadSeeker = prs(0)
// New creates an instance of AuthCenter.
func New(conns pool.Pool, key *keys.PrivateKey, config *tokens.CacheConfig) Center {
func New(conns pool.Pool, key *keys.PrivateKey, config *cache.Config) Center {
return &center{
cli: tokens.New(conns, key, config),
reg: &regexpSubmatcher{re: authorizationFieldRegexp},

38
creds/tokens/accessbox_cache.go → api/cache/accessbox.go vendored
View file

@ -1,4 +1,4 @@
package tokens
package cache
import (
"time"
@ -8,31 +8,33 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
)
const (
// DefaultCacheSize is a default maximum number of entries in cache.
DefaultCacheSize = 100
// DefaultCacheLifetime is a default lifetime of entries in cache.
DefaultCacheLifetime = 10 * time.Minute
)
type (
// AccessBoxCache stores access box by its address.
AccessBoxCache struct {
cache gcache.Cache
}
// CacheConfig stores expiration params for cache.
type CacheConfig struct {
// Config stores expiration params for cache.
Config struct {
Size int
Lifetime time.Duration
}
}
)
// DefaultCacheConfig return new default cache expiration values.
func DefaultCacheConfig() *CacheConfig {
return &CacheConfig{Size: DefaultCacheSize, Lifetime: DefaultCacheLifetime}
}
const (
// DefaultAccessBoxCacheSize is a default maximum number of entries in cache.
DefaultAccessBoxCacheSize = 100
// DefaultAccessBoxCacheLifetime is a default lifetime of entries in cache.
DefaultAccessBoxCacheLifetime = 10 * time.Minute
)
// AccessBoxCache stores access box by its address.
type AccessBoxCache struct {
cache gcache.Cache
// DefaultAccessBoxConfig return new default cache expiration values.
func DefaultAccessBoxConfig() *Config {
return &Config{Size: DefaultAccessBoxCacheSize, Lifetime: DefaultAccessBoxCacheLifetime}
}
// NewAccessBoxCache creates an object of BucketCache.
func NewAccessBoxCache(config *CacheConfig) *AccessBoxCache {
func NewAccessBoxCache(config *Config) *AccessBoxCache {
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
return &AccessBoxCache{cache: gc}

45
api/cache/buckets.go vendored
View file

@ -4,39 +4,40 @@ import (
"time"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
)
type (
// BucketCache provides interface for lru cache for objects.
BucketCache interface {
Get(key string) *api.BucketInfo
Put(bkt *api.BucketInfo) error
Delete(key string) bool
}
// GetBucketCache contains cache with objects and lifetime of cache entries.
GetBucketCache struct {
// BucketCache contains cache with objects and lifetime of cache entries.
type BucketCache struct {
cache gcache.Cache
lifetime time.Duration
}
}
const (
// DefaultBucketCacheSize is a default maximum number of entries in cache.
DefaultBucketCacheSize = 1e3
// DefaultBucketCacheLifetime is a default lifetime of entries in cache.
DefaultBucketCacheLifetime = time.Minute
)
// DefaultBucketConfig return new default cache expiration values.
func DefaultBucketConfig() *Config {
return &Config{Size: DefaultBucketCacheSize, Lifetime: DefaultBucketCacheLifetime}
}
// NewBucketCache creates an object of BucketCache.
func NewBucketCache(cacheSize int, lifetime time.Duration) *GetBucketCache {
gc := gcache.New(cacheSize).LRU().Build()
return &GetBucketCache{cache: gc, lifetime: lifetime}
func NewBucketCache(config *Config) *BucketCache {
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
return &BucketCache{cache: gc}
}
// Get returns cached object.
func (o *GetBucketCache) Get(key string) *api.BucketInfo {
func (o *BucketCache) Get(key string) *data.BucketInfo {
entry, err := o.cache.Get(key)
if err != nil {
return nil
}
result, ok := entry.(*api.BucketInfo)
result, ok := entry.(*data.BucketInfo)
if !ok {
return nil
}
@ -45,11 +46,11 @@ func (o *GetBucketCache) Get(key string) *api.BucketInfo {
}
// Put puts an object to cache.
func (o *GetBucketCache) Put(bkt *api.BucketInfo) error {
return o.cache.SetWithExpire(bkt.Name, bkt, o.lifetime)
func (o *BucketCache) Put(bkt *data.BucketInfo) error {
return o.cache.Set(bkt.Name, bkt)
}
// Delete deletes an object from cache.
func (o *GetBucketCache) Delete(key string) bool {
func (o *BucketCache) Delete(key string) bool {
return o.cache.Remove(key)
}

39
api/cache/names.go vendored
View file

@ -7,32 +7,33 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object"
)
// ObjectsNameCache provides interface for lru cache for objects.
// ObjectsNameCache provides for lru cache for objects.
// This cache contains mapping nice name to object addresses.
// Key is bucketName+objectName.
type ObjectsNameCache interface {
Get(key string) *object.Address
Put(key string, address *object.Address) error
Delete(key string) bool
type ObjectsNameCache struct {
cache gcache.Cache
}
type (
// NameCache contains cache with objects and lifetime of cache entries.
NameCache struct {
cache gcache.Cache
lifetime time.Duration
}
const (
// DefaultObjectsNameCacheSize is a default maximum number of entries in cache.
DefaultObjectsNameCacheSize = 1e4
// DefaultObjectsNameCacheLifetime is a default lifetime of entries in cache.
DefaultObjectsNameCacheLifetime = time.Minute
)
// NewObjectsNameCache creates an object of ObjectsNameCache.
func NewObjectsNameCache(cacheSize int, lifetime time.Duration) *NameCache {
gc := gcache.New(cacheSize).LRU().Build()
// DefaultObjectsNameConfig return new default cache expiration values.
func DefaultObjectsNameConfig() *Config {
return &Config{Size: DefaultObjectsNameCacheSize, Lifetime: DefaultObjectsNameCacheLifetime}
}
return &NameCache{cache: gc, lifetime: lifetime}
// NewObjectsNameCache creates an object of ObjectsNameCache.
func NewObjectsNameCache(config *Config) *ObjectsNameCache {
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
return &ObjectsNameCache{cache: gc}
}
// Get returns cached object.
func (o *NameCache) Get(key string) *object.Address {
func (o *ObjectsNameCache) Get(key string) *object.Address {
entry, err := o.cache.Get(key)
if err != nil {
return nil
@ -47,11 +48,11 @@ func (o *NameCache) Get(key string) *object.Address {
}
// Put puts an object to cache.
func (o *NameCache) Put(key string, address *object.Address) error {
return o.cache.SetWithExpire(key, address, o.lifetime)
func (o *ObjectsNameCache) Put(key string, address *object.Address) error {
return o.cache.Set(key, address)
}
// Delete deletes an object from cache.
func (o *NameCache) Delete(key string) bool {
func (o *ObjectsNameCache) Delete(key string) bool {
return o.cache.Remove(key)
}

34
api/cache/objects.go vendored
View file

@ -7,11 +7,9 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object"
)
// ObjectsCache provides interface for lru cache for objects.
type ObjectsCache interface {
Get(address *object.Address) *object.Object
Put(obj object.Object) error
Delete(address *object.Address) bool
// ObjectsCache provides lru cache for objects.
type ObjectsCache struct {
cache gcache.Cache
}
const (
@ -21,23 +19,19 @@ const (
DefaultObjectsCacheSize = 1e6
)
type (
// ObjectHeadersCache contains cache with objects and lifetime of cache entries.
ObjectHeadersCache struct {
cache gcache.Cache
lifetime time.Duration
}
)
// DefaultObjectsConfig return new default cache expiration values.
func DefaultObjectsConfig() *Config {
return &Config{Size: DefaultObjectsCacheSize, Lifetime: DefaultObjectsCacheLifetime}
}
// New creates an object of ObjectHeadersCache.
func New(cacheSize int, lifetime time.Duration) *ObjectHeadersCache {
gc := gcache.New(cacheSize).LRU().Build()
return &ObjectHeadersCache{cache: gc, lifetime: lifetime}
func New(config *Config) *ObjectsCache {
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
return &ObjectsCache{cache: gc}
}
// Get returns cached object.
func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object {
func (o *ObjectsCache) Get(address *object.Address) *object.Object {
entry, err := o.cache.Get(address.String())
if err != nil {
return nil
@ -52,11 +46,11 @@ func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object {
}
// Put puts an object to cache.
func (o *ObjectHeadersCache) Put(obj object.Object) error {
return o.cache.SetWithExpire(obj.ContainerID().String()+"/"+obj.ID().String(), obj, o.lifetime)
func (o *ObjectsCache) Put(obj object.Object) error {
return o.cache.Set(obj.ContainerID().String()+"/"+obj.ID().String(), obj)
}
// Delete deletes an object from cache.
func (o *ObjectHeadersCache) Delete(address *object.Address) bool {
func (o *ObjectsCache) Delete(address *object.Address) bool {
return o.cache.Remove(address.String())
}

14
api/cache/objects_test.go vendored
View file

@ -9,10 +9,12 @@ import (
"github.com/stretchr/testify/require"
)
const (
cachesize = 10
lifetime = time.Second * 5
)
func getTestConfig() *Config {
return &Config{
Size: 10,
Lifetime: 5 * time.Second,
}
}
func TestCache(t *testing.T) {
obj := objecttest.Object()
@ -21,7 +23,7 @@ func TestCache(t *testing.T) {
address.SetObjectID(obj.ID())
t.Run("check get", func(t *testing.T) {
cache := New(cachesize, lifetime)
cache := New(getTestConfig())
err := cache.Put(*obj)
require.NoError(t, err)
@ -30,7 +32,7 @@ func TestCache(t *testing.T) {
})
t.Run("check delete", func(t *testing.T) {
cache := New(cachesize, lifetime)
cache := New(getTestConfig())
err := cache.Put(*obj)
require.NoError(t, err)

51
api/cache/objectslist.go vendored
View file

@ -24,25 +24,8 @@ import (
*/
type (
// ObjectsListCache provides interface for cache of ListObjectsV2 in a layer struct.
ObjectsListCache interface {
Get(key ObjectsListKey) []*object.ID
Put(key ObjectsListKey, oids []*object.ID) error
CleanCacheEntriesContainingObject(objectName string, cid *cid.ID)
}
)
const (
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
DefaultObjectsListCacheLifetime = time.Second * 60
// DefaultObjectsListCacheSize is a default size of cache of ListObjects.
DefaultObjectsListCacheSize = 1e5
)
type (
// ListObjectsCache contains cache for ListObjects and ListObjectVersions.
ListObjectsCache struct {
lifetime time.Duration
// ObjectsListCache contains cache for ListObjects and ListObjectVersions.
ObjectsListCache struct {
cache gcache.Cache
}
@ -53,18 +36,26 @@ type (
}
)
// NewObjectsListCache is a constructor which creates an object of ListObjectsCache with given lifetime of entries.
func NewObjectsListCache(cacheSize int, lifetime time.Duration) *ListObjectsCache {
gc := gcache.New(cacheSize).LRU().Build()
const (
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
DefaultObjectsListCacheLifetime = time.Second * 60
// DefaultObjectsListCacheSize is a default size of cache of ListObjects.
DefaultObjectsListCacheSize = 1e5
)
return &ListObjectsCache{
cache: gc,
lifetime: lifetime,
}
// DefaultObjectsListConfig return new default cache expiration values.
func DefaultObjectsListConfig() *Config {
return &Config{Size: DefaultObjectsListCacheSize, Lifetime: DefaultObjectsListCacheLifetime}
}
// NewObjectsListCache is a constructor which creates an object of ListObjectsCache with given lifetime of entries.
func NewObjectsListCache(config *Config) *ObjectsListCache {
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
return &ObjectsListCache{cache: gc}
}
// Get return list of ObjectInfo.
func (l *ListObjectsCache) Get(key ObjectsListKey) []*object.ID {
func (l *ObjectsListCache) Get(key ObjectsListKey) []*object.ID {
entry, err := l.cache.Get(key)
if err != nil {
return nil
@ -79,16 +70,16 @@ func (l *ListObjectsCache) Get(key ObjectsListKey) []*object.ID {
}
// Put puts a list of objects to cache.
func (l *ListObjectsCache) Put(key ObjectsListKey, oids []*object.ID) error {
func (l *ObjectsListCache) Put(key ObjectsListKey, oids []*object.ID) error {
if len(oids) == 0 {
return fmt.Errorf("list is empty, cid: %s, prefix: %s", key.cid, key.prefix)
}
return l.cache.SetWithExpire(key, oids, l.lifetime)
return l.cache.Set(key, oids)
}
// CleanCacheEntriesContainingObject deletes entries containing specified object.
func (l *ListObjectsCache) CleanCacheEntriesContainingObject(objectName string, cid *cid.ID) {
func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string, cid *cid.ID) {
cidStr := cid.String()
keys := l.cache.Keys(true)
for _, key := range keys {

32
api/cache/objectslist_test.go vendored
View file

@ -14,6 +14,13 @@ import (
const testingCacheLifetime = 5 * time.Second
const testingCacheSize = 10
func getTestObjectsListConfig() *Config {
return &Config{
Size: testingCacheSize,
Lifetime: testingCacheLifetime,
}
}
func randID(t *testing.T) *object.ID {
id := object.NewID()
id.SetSHA256(randSHA256Checksum(t))
@ -42,7 +49,8 @@ func TestObjectsListCache(t *testing.T) {
t.Run("lifetime", func(t *testing.T) {
var (
cache = NewObjectsListCache(testingCacheSize, testingCacheLifetime)
config = getTestObjectsListConfig()
cache = NewObjectsListCache(config)
cacheKey = ObjectsListKey{cid: userKey}
)
@ -53,13 +61,13 @@ func TestObjectsListCache(t *testing.T) {
return cache.Get(cacheKey) == nil
}
require.Never(t, condition, cache.lifetime, time.Second)
require.Never(t, condition, config.Lifetime, time.Second)
require.Eventually(t, condition, time.Second, 10*time.Millisecond)
})
t.Run("get cache with empty prefix", func(t *testing.T) {
var (
cache = NewObjectsListCache(testingCacheSize, testingCacheLifetime)
cache = NewObjectsListCache(getTestObjectsListConfig())
cacheKey = ObjectsListKey{cid: userKey}
)
err := cache.Put(cacheKey, ids)
@ -79,7 +87,7 @@ func TestObjectsListCache(t *testing.T) {
prefix: "dir",
}
cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime)
cache := NewObjectsListCache(getTestObjectsListConfig())
err := cache.Put(cacheKey, ids)
require.NoError(t, err)
@ -104,7 +112,7 @@ func TestObjectsListCache(t *testing.T) {
}
)
cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime)
cache := NewObjectsListCache(getTestObjectsListConfig())
err := cache.Put(cacheKey, ids)
require.NoError(t, err)
@ -122,7 +130,7 @@ func TestObjectsListCache(t *testing.T) {
}
)
cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime)
cache := NewObjectsListCache(getTestObjectsListConfig())
err := cache.Put(cacheKey, ids)
require.NoError(t, err)
@ -143,7 +151,9 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) {
}
t.Run("put object to the root of the bucket", func(t *testing.T) {
cache := NewObjectsListCache(testingCacheSize, time.Minute)
config := getTestObjectsListConfig()
config.Lifetime = time.Minute
cache := NewObjectsListCache(config)
for _, k := range keys {
err := cache.Put(k, oids)
require.NoError(t, err)
@ -160,7 +170,9 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) {
})
t.Run("put object to dir/", func(t *testing.T) {
cache := NewObjectsListCache(testingCacheSize, time.Minute)
config := getTestObjectsListConfig()
config.Lifetime = time.Minute
cache := NewObjectsListCache(config)
for _, k := range keys {
err := cache.Put(k, oids)
require.NoError(t, err)
@ -177,7 +189,9 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) {
})
t.Run("put object to dir/lol/", func(t *testing.T) {
cache := NewObjectsListCache(testingCacheSize, time.Minute)
config := getTestObjectsListConfig()
config.Lifetime = time.Minute
cache := NewObjectsListCache(config)
for _, k := range keys {
err := cache.Put(k, oids)
require.NoError(t, err)

45
api/cache/system.go vendored
View file

@ -7,32 +7,33 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object"
)
type (
// SystemCache provides interface for lru cache for objects.
// This cache contains "system" objects (bucket versioning settings, tagging object etc.).
// Key is bucketName+systemFileName.
SystemCache interface {
Get(key string) *object.Object
Put(key string, obj *object.Object) error
Delete(key string) bool
}
// SysCache contains cache with objects and lifetime of cache entries.
SysCache struct {
// SystemCache provides lru cache for objects.
// This cache contains "system" objects (bucket versioning settings, tagging object etc.).
// Key is bucketName+systemFileName.
type SystemCache struct {
cache gcache.Cache
lifetime time.Duration
}
}
const (
// DefaultSystemCacheSize is a default maximum number of entries in cache.
DefaultSystemCacheSize = 1e4
// DefaultSystemCacheLifetime is a default lifetime of entries in cache.
DefaultSystemCacheLifetime = 5 * time.Minute
)
// NewSystemCache creates an object of SystemCache.
func NewSystemCache(cacheSize int, lifetime time.Duration) *SysCache {
gc := gcache.New(cacheSize).LRU().Build()
// DefaultSystemConfig return new default cache expiration values.
func DefaultSystemConfig() *Config {
return &Config{Size: DefaultSystemCacheSize, Lifetime: DefaultSystemCacheLifetime}
}
return &SysCache{cache: gc, lifetime: lifetime}
// NewSystemCache creates an object of SystemCache.
func NewSystemCache(config *Config) *SystemCache {
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
return &SystemCache{cache: gc}
}
// Get returns cached object.
func (o *SysCache) Get(key string) *object.Object {
func (o *SystemCache) Get(key string) *object.Object {
entry, err := o.cache.Get(key)
if err != nil {
return nil
@ -47,11 +48,11 @@ func (o *SysCache) Get(key string) *object.Object {
}
// Put puts an object to cache.
func (o *SysCache) Put(key string, obj *object.Object) error {
return o.cache.SetWithExpire(key, obj, o.lifetime)
func (o *SystemCache) Put(key string, obj *object.Object) error {
return o.cache.Set(key, obj)
}
// Delete deletes an object from cache.
func (o *SysCache) Delete(key string) bool {
func (o *SystemCache) Delete(key string) bool {
return o.cache.Remove(key)
}

2
api/info.go → api/data/info.go
View file

@ -1,4 +1,4 @@
package api
package data
import (
"time"

3
api/handler/acl.go
View file

@ -14,6 +14,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/v2/acl"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
)
@ -299,7 +300,7 @@ func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
}
}
func checkOwner(info *api.BucketInfo, owner string) error {
func checkOwner(info *data.BucketInfo, owner string) error {
if owner == "" {
return nil
}

3
api/handler/copy.go
View file

@ -7,6 +7,7 @@ import (
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
@ -32,7 +33,7 @@ func path2BucketObject(path string) (bucket, prefix string) {
func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
info *api.ObjectInfo
info *data.ObjectInfo
metadata map[string]string
reqInfo = api.GetReqInfo(r.Context())

7
api/handler/get.go
View file

@ -8,6 +8,7 @@ import (
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
)
@ -65,7 +66,7 @@ func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams,
return &layer.RangeParams{Start: start, End: end}, nil
}
func writeHeaders(h http.Header, info *api.ObjectInfo, tagSetLength int) {
func writeHeaders(h http.Header, info *data.ObjectInfo, tagSetLength int) {
if len(info.ContentType) > 0 {
h.Set(api.ContentType, info.ContentType)
}
@ -83,7 +84,7 @@ func writeHeaders(h http.Header, info *api.ObjectInfo, tagSetLength int) {
func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
info *api.ObjectInfo
info *data.ObjectInfo
params *layer.RangeParams
reqInfo = api.GetReqInfo(r.Context())
@ -143,7 +144,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
}
}
func checkPreconditions(info *api.ObjectInfo, args *conditionalArgs) error {
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error {
if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum {
return errors.GetAPIError(errors.ErrPreconditionFailed)
}

10
api/handler/get_test.go
View file

@ -5,7 +5,7 @@ import (
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
@ -46,8 +46,8 @@ func TestFetchRangeHeader(t *testing.T) {
}
}
func newInfo(etag string, created time.Time) *api.ObjectInfo {
return &api.ObjectInfo{
func newInfo(etag string, created time.Time) *data.ObjectInfo {
return &data.ObjectInfo{
HashSum: etag,
Created: created,
}
@ -61,13 +61,13 @@ func TestPreconditions(t *testing.T) {
for _, tc := range []struct {
name string
info *api.ObjectInfo
info *data.ObjectInfo
args *conditionalArgs
expected error
}{
{
name: "no conditions",
info: new(api.ObjectInfo),
info: new(data.ObjectInfo),
args: new(conditionalArgs),
expected: nil,
},

3
api/handler/head.go
View file

@ -5,6 +5,7 @@ import (
"net/http"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
@ -27,7 +28,7 @@ func getRangeToDetectContentType(maxSize int64) *layer.RangeParams {
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
info *api.ObjectInfo
info *data.ObjectInfo
reqInfo = api.GetReqInfo(r.Context())
)

5
api/handler/object_list.go
View file

@ -8,6 +8,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
)
@ -183,11 +184,11 @@ func fillPrefixes(src []string, encode string) []CommonPrefix {
return dst
}
func fillContentsWithOwner(src []*api.ObjectInfo, encode string) []Object {
func fillContentsWithOwner(src []*data.ObjectInfo, encode string) []Object {
return fillContents(src, encode, true)
}
func fillContents(src []*api.ObjectInfo, encode string, fetchOwner bool) []Object {
func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Object {
var dst []Object
for _, obj := range src {
res := Object{

3
api/handler/put.go
View file

@ -17,6 +17,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
@ -419,7 +420,7 @@ func containsACLHeaders(r *http.Request) bool {
r.Header.Get(api.AmzGrantFullControl) != "" || r.Header.Get(api.AmzGrantWrite) != ""
}
func (h *handler) getNewEAclTable(r *http.Request, objInfo *api.ObjectInfo) (*eacl.Table, error) {
func (h *handler) getNewEAclTable(r *http.Request, objInfo *data.ObjectInfo) (*eacl.Table, error) {
var newEaclTable *eacl.Table
objectACL, err := parseACLHeaders(r)
if err != nil {

13
api/layer/container.go
View file

@ -12,6 +12,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/container"
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"go.uber.org/zap"
@ -20,19 +21,19 @@ import (
type (
// BucketACL extends BucketInfo by eacl.Table.
BucketACL struct {
Info *api.BucketInfo
Info *data.BucketInfo
EACL *eacl.Table
}
)
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*api.BucketInfo, error) {
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*data.BucketInfo, error) {
var (
err error
res *container.Container
rid = api.GetRequestID(ctx)
bearerOpt = n.BearerOpt(ctx)
info = &api.BucketInfo{
info = &data.BucketInfo{
CID: cid,
Name: cid.String(),
}
@ -83,7 +84,7 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*api.BucketInfo
return info, nil
}
func (n *layer) containerList(ctx context.Context) ([]*api.BucketInfo, error) {
func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
var (
err error
own = n.Owner(ctx)
@ -99,7 +100,7 @@ func (n *layer) containerList(ctx context.Context) ([]*api.BucketInfo, error) {
return nil, err
}
list := make([]*api.BucketInfo, 0, len(res))
list := make([]*data.BucketInfo, 0, len(res))
for _, cid := range res {
info, err := n.containerInfo(ctx, cid)
if err != nil {
@ -117,7 +118,7 @@ func (n *layer) containerList(ctx context.Context) ([]*api.BucketInfo, error) {
func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*cid.ID, error) {
var err error
bktInfo := &api.BucketInfo{
bktInfo := &data.BucketInfo{
Name: p.Name,
Owner: n.Owner(ctx),
Created: time.Now(),

100
api/layer/layer.go
View file

@ -19,6 +19,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/pool"
@ -29,19 +30,20 @@ type (
layer struct {
pool pool.Pool
log *zap.Logger
listsCache cache.ObjectsListCache
objCache cache.ObjectsCache
namesCache cache.ObjectsNameCache
bucketCache cache.BucketCache
systemCache cache.SystemCache
listsCache *cache.ObjectsListCache
objCache *cache.ObjectsCache
namesCache *cache.ObjectsNameCache
bucketCache *cache.BucketCache
systemCache *cache.SystemCache
}
// CacheConfig contains params for caches.
CacheConfig struct {
Lifetime time.Duration
Size int
ListObjectsLifetime time.Duration
ListObjectsSize int
// CachesConfig contains params for caches.
CachesConfig struct {
Objects *cache.Config
ObjectsList *cache.Config
Names *cache.Config
Buckets *cache.Config
System *cache.Config
}
// Params stores basic API parameters.
@ -55,7 +57,7 @@ type (
// GetObjectParams stores object get request parameters.
GetObjectParams struct {
Range *RangeParams
ObjectInfo *api.ObjectInfo
ObjectInfo *data.ObjectInfo
Offset int64
Length int64
Writer io.Writer
@ -97,7 +99,7 @@ type (
// CopyObjectParams stores object copy request parameters.
CopyObjectParams struct {
SrcObject *api.ObjectInfo
SrcObject *data.ObjectInfo
DstBucket string
DstObject string
SrcSize int64
@ -141,7 +143,7 @@ type (
// PutTaggingParams stores tag set params.
PutTaggingParams struct {
ObjectInfo *api.ObjectInfo
ObjectInfo *data.ObjectInfo
TagSet map[string]string
}
@ -154,33 +156,33 @@ type (
Client interface {
NeoFS
PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*api.ObjectInfo, error)
PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*data.ObjectInfo, error)
GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error)
ListBuckets(ctx context.Context) ([]*api.BucketInfo, error)
GetBucketInfo(ctx context.Context, name string) (*api.BucketInfo, error)
ListBuckets(ctx context.Context) ([]*data.BucketInfo, error)
GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error)
GetBucketACL(ctx context.Context, name string) (*BucketACL, error)
PutBucketACL(ctx context.Context, p *PutBucketACLParams) error
CreateBucket(ctx context.Context, p *CreateBucketParams) (*cid.ID, error)
DeleteBucket(ctx context.Context, p *DeleteBucketParams) error
GetObject(ctx context.Context, p *GetObjectParams) error
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*api.ObjectInfo, error)
GetObjectTagging(ctx context.Context, p *api.ObjectInfo) (map[string]string, error)
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error)
GetObjectTagging(ctx context.Context, p *data.ObjectInfo) (map[string]string, error)
GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error)
PutObject(ctx context.Context, p *PutObjectParams) (*api.ObjectInfo, error)
PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error)
PutObjectTagging(ctx context.Context, p *PutTaggingParams) error
PutBucketTagging(ctx context.Context, bucket string, tagSet map[string]string) error
CopyObject(ctx context.Context, p *CopyObjectParams) (*api.ObjectInfo, error)
CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ObjectInfo, error)
ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error)
ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error)
ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) ([]*VersionedObject, error)
DeleteObjectTagging(ctx context.Context, p *api.ObjectInfo) error
DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) error
DeleteBucketTagging(ctx context.Context, bucket string) error
}
)
@ -194,18 +196,28 @@ func (t *VersionedObject) String() string {
return t.Name + ":" + t.VersionID
}
// DefaultCachesConfigs returns filled configs.
func DefaultCachesConfigs() *CachesConfig {
return &CachesConfig{
Objects: cache.DefaultObjectsConfig(),
ObjectsList: cache.DefaultObjectsListConfig(),
Names: cache.DefaultObjectsNameConfig(),
Buckets: cache.DefaultBucketConfig(),
System: cache.DefaultSystemConfig(),
}
}
// NewLayer creates instance of layer. It checks credentials
// and establishes gRPC connection with node.
func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
func NewLayer(log *zap.Logger, conns pool.Pool, config *CachesConfig) Client {
return &layer{
pool: conns,
log: log,
listsCache: cache.NewObjectsListCache(config.ListObjectsSize, config.ListObjectsLifetime),
objCache: cache.New(config.Size, config.Lifetime),
//todo reconsider cache params
namesCache: cache.NewObjectsNameCache(1000, time.Minute),
bucketCache: cache.NewBucketCache(150, time.Minute),
systemCache: cache.NewSystemCache(1000, 5*time.Minute),
listsCache: cache.NewObjectsListCache(config.ObjectsList),
objCache: cache.New(config.Objects),
namesCache: cache.NewObjectsNameCache(config.Names),
bucketCache: cache.NewBucketCache(config.Buckets),
systemCache: cache.NewSystemCache(config.System),
}
}
@ -243,7 +255,7 @@ func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Objec
}
// GetBucketInfo returns bucket info by name.
func (n *layer) GetBucketInfo(ctx context.Context, name string) (*api.BucketInfo, error) {
func (n *layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error) {
name, err := url.QueryUnescape(name)
if err != nil {
return nil, err
@ -301,7 +313,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err
// ListBuckets returns all user containers. Name of the bucket is a container
// id. Timestamp is omitted since it is not saved in neofs container.
func (n *layer) ListBuckets(ctx context.Context) ([]*api.BucketInfo, error) {
func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
return n.containerList(ctx)
}
@ -337,7 +349,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
}
// GetObjectInfo returns meta information about the object.
func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*api.ObjectInfo, error) {
func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) {
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
if err != nil {
n.log.Error("could not fetch bucket info", zap.Error(err))
@ -352,7 +364,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*api.Ob
}
// PutObject into storage.
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*api.ObjectInfo, error) {
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) {
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
if err != nil {
return nil, err
@ -362,8 +374,8 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*api.ObjectI
}
// GetObjectTagging from storage.
func (n *layer) GetObjectTagging(ctx context.Context, oi *api.ObjectInfo) (map[string]string, error) {
bktInfo := &api.BucketInfo{
func (n *layer) GetObjectTagging(ctx context.Context, oi *data.ObjectInfo) (map[string]string, error) {
bktInfo := &data.BucketInfo{
Name: oi.Bucket,
CID: oi.CID,
Owner: oi.Owner,
@ -392,7 +404,7 @@ func (n *layer) GetBucketTagging(ctx context.Context, bucketName string) (map[st
return formTagSet(objInfo), nil
}
func formTagSet(objInfo *api.ObjectInfo) map[string]string {
func formTagSet(objInfo *data.ObjectInfo) map[string]string {
var tagSet map[string]string
if objInfo != nil {
tagSet = make(map[string]string, len(objInfo.Headers))
@ -410,7 +422,7 @@ func formTagSet(objInfo *api.ObjectInfo) map[string]string {
// PutObjectTagging into storage.
func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error {
bktInfo := &api.BucketInfo{
bktInfo := &data.BucketInfo{
Name: p.ObjectInfo.Bucket,
CID: p.ObjectInfo.CID,
Owner: p.ObjectInfo.Owner,
@ -438,7 +450,7 @@ func (n *layer) PutBucketTagging(ctx context.Context, bucketName string, tagSet
}
// DeleteObjectTagging from storage.
func (n *layer) DeleteObjectTagging(ctx context.Context, p *api.ObjectInfo) error {
func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) error {
bktInfo, err := n.GetBucketInfo(ctx, p.Bucket)
if err != nil {
return err
@ -446,7 +458,7 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *api.ObjectInfo) erro
return n.deleteSystemObject(ctx, bktInfo, p.TagsObject())
}
func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *api.BucketInfo, name string) error {
func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error {
var oid *object.ID
if meta := n.systemCache.Get(bktInfo.SystemObjectKey(name)); meta != nil {
oid = meta.ID()
@ -475,7 +487,7 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, bucketName string) erro
return n.deleteSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName))
}
func (n *layer) putSystemObject(ctx context.Context, bktInfo *api.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.Object, error) {
func (n *layer) putSystemObject(ctx context.Context, bktInfo *data.BucketInfo, objName string, metadata map[string]string, prefix string) (*object.Object, error) {
var (
err error
oldOID *object.ID
@ -542,7 +554,7 @@ func (n *layer) putSystemObject(ctx context.Context, bktInfo *api.BucketInfo, ob
return meta, nil
}
func (n *layer) getSystemObject(ctx context.Context, bkt *api.BucketInfo, objName string) (*api.ObjectInfo, error) {
func (n *layer) getSystemObject(ctx context.Context, bkt *data.BucketInfo, objName string) (*data.ObjectInfo, error) {
if meta := n.systemCache.Get(bkt.SystemObjectKey(objName)); meta != nil {
return objInfoFromMeta(bkt, meta), nil
}
@ -564,7 +576,7 @@ func (n *layer) getSystemObject(ctx context.Context, bkt *api.BucketInfo, objNam
}
// CopyObject from one bucket into another bucket.
func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*api.ObjectInfo, error) {
func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ObjectInfo, error) {
pr, pw := io.Pipe()
go func() {
@ -588,7 +600,7 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*api.Objec
}
// DeleteObject removes all objects with passed nice name.
func (n *layer) deleteObject(ctx context.Context, bkt *api.BucketInfo, obj *VersionedObject) *VersionedObject {
func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) *VersionedObject {
var (
err error
ids []*object.ID
@ -642,7 +654,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *api.BucketInfo, obj *Vers
obj.Error = err
return obj
}
if err = n.DeleteObjectTagging(ctx, &api.ObjectInfo{ID: id, Bucket: bkt.Name, Name: obj.Name}); err != nil {
if err = n.DeleteObjectTagging(ctx, &data.ObjectInfo{ID: id, Bucket: bkt.Name, Name: obj.Name}); err != nil {
obj.Error = err
return obj
}

41
api/layer/object.go
View file

@ -16,6 +16,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"go.uber.org/zap"
)
@ -62,7 +63,7 @@ type (
}
allObjectParams struct {
Bucket *api.BucketInfo
Bucket *data.BucketInfo
Delimiter string
Prefix string
}
@ -134,7 +135,7 @@ func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) {
}
// objectPut into NeoFS, took payload from io.Reader.
func (n *layer) objectPut(ctx context.Context, bkt *api.BucketInfo, p *PutObjectParams) (*api.ObjectInfo, error) {
func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjectParams) (*data.ObjectInfo, error) {
own := n.Owner(ctx)
obj, err := url.QueryUnescape(p.Object)
if err != nil {
@ -198,7 +199,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *api.BucketInfo, p *PutObject
}
}
return &api.ObjectInfo{
return &data.ObjectInfo{
ID: oid,
CID: bkt.CID,
@ -287,7 +288,7 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio
return idsToDeleteArr
}
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *api.BucketInfo, objectName string) (*api.ObjectInfo, error) {
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ObjectInfo, error) {
if address := n.namesCache.Get(bkt.Name + "/" + objectName); address != nil {
if headInfo := n.objCache.Get(address); headInfo != nil {
return objInfoFromMeta(bkt, headInfo), nil
@ -313,7 +314,7 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *api.Bucket
return lastVersion, nil
}
func (n *layer) headVersions(ctx context.Context, bkt *api.BucketInfo, objectName string) (*objectVersions, error) {
func (n *layer) headVersions(ctx context.Context, bkt *data.BucketInfo, objectName string) (*objectVersions, error) {
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName})
if err != nil {
return nil, err
@ -351,7 +352,7 @@ func (n *layer) headVersions(ctx context.Context, bkt *api.BucketInfo, objectNam
return versions, nil
}
func (n *layer) headVersion(ctx context.Context, bkt *api.BucketInfo, versionID string) (*api.ObjectInfo, error) {
func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, versionID string) (*data.ObjectInfo, error) {
oid := object.NewID()
if err := oid.Parse(versionID); err != nil {
return nil, err
@ -396,7 +397,7 @@ func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*Lis
var (
err error
result ListObjectsInfoV1
allObjects []*api.ObjectInfo
allObjects []*data.ObjectInfo
)
if p.MaxKeys == 0 {
@ -431,7 +432,7 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
var (
err error
result ListObjectsInfoV2
allObjects []*api.ObjectInfo
allObjects []*data.ObjectInfo
)
if p.MaxKeys == 0 {
@ -465,13 +466,13 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
return &result, nil
}
func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*api.ObjectInfo, error) {
func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*data.ObjectInfo, error) {
versions, err := n.getAllObjectsVersions(ctx, p.Bucket, p.Prefix, p.Delimiter)
if err != nil {
return nil, err
}
objects := make([]*api.ObjectInfo, 0, len(versions))
objects := make([]*data.ObjectInfo, 0, len(versions))
for _, v := range versions {
lastVersion := v.getLast()
if lastVersion != nil {
@ -486,7 +487,7 @@ func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*ap
return objects, nil
}
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *api.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) {
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) {
var err error
cacheKey := cache.CreateObjectsListCacheKey(bkt.CID, prefix)
@ -534,12 +535,12 @@ func splitVersions(header string) []string {
return strings.Split(header, ",")
}
func isSystem(obj *api.ObjectInfo) bool {
func isSystem(obj *data.ObjectInfo) bool {
return len(obj.Headers[objectSystemAttributeName]) > 0 ||
len(obj.Headers[attrVersionsIgnore]) > 0
}
func trimAfterObjectName(startAfter string, objects []*api.ObjectInfo) []*api.ObjectInfo {
func trimAfterObjectName(startAfter string, objects []*data.ObjectInfo) []*data.ObjectInfo {
if len(objects) != 0 && objects[len(objects)-1].Name <= startAfter {
return nil
}
@ -552,9 +553,9 @@ func trimAfterObjectName(startAfter string, objects []*api.ObjectInfo) []*api.Ob
return nil
}
func trimAfterObjectID(id string, objects []*api.ObjectInfo) []*api.ObjectInfo {
func trimAfterObjectID(id string, objects []*data.ObjectInfo) []*data.ObjectInfo {
if len(objects) != 0 && objects[len(objects)-1].ID.String() == id {
return []*api.ObjectInfo{}
return []*data.ObjectInfo{}
}
for i, obj := range objects {
if obj.ID.String() == id {
@ -565,7 +566,7 @@ func trimAfterObjectID(id string, objects []*api.ObjectInfo) []*api.ObjectInfo {
return nil
}
func triageObjects(allObjects []*api.ObjectInfo) (prefixes []string, objects []*api.ObjectInfo) {
func triageObjects(allObjects []*data.ObjectInfo) (prefixes []string, objects []*data.ObjectInfo) {
for _, ov := range allObjects {
if ov.IsDir {
prefixes = append(prefixes, ov.Name)
@ -577,11 +578,11 @@ func triageObjects(allObjects []*api.ObjectInfo) (prefixes []string, objects []*
return
}
func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*api.ObjectInfo, error) {
func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*data.ObjectInfo, error) {
var (
err error
bkt *api.BucketInfo
allObjects []*api.ObjectInfo
bkt *data.BucketInfo
allObjects []*data.ObjectInfo
)
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
@ -600,7 +601,7 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
return allObjects, nil
}
func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *api.BucketInfo) bool {
func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *data.BucketInfo) bool {
settings, err := n.getBucketSettings(ctx, bktInfo)
if err != nil {
n.log.Warn("couldn't get versioning settings object", zap.Error(err))

12
api/layer/object_test.go
View file

@ -6,7 +6,7 @@ import (
"testing"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/stretchr/testify/require"
)
@ -26,11 +26,11 @@ func randSHA256Checksum(t *testing.T) (cs [sha256.Size]byte) {
func TestTrimAfterObjectName(t *testing.T) {
var (
objects []*api.ObjectInfo
objects []*data.ObjectInfo
names = []string{"b", "c", "d"}
)
for _, name := range names {
objects = append(objects, &api.ObjectInfo{Name: name})
objects = append(objects, &data.ObjectInfo{Name: name})
}
t.Run("startafter before all objects", func(t *testing.T) {
@ -59,7 +59,7 @@ func TestTrimAfterObjectName(t *testing.T) {
})
t.Run("empty objects", func(t *testing.T) {
actual := trimAfterObjectName(names[0], []*api.ObjectInfo{})
actual := trimAfterObjectName(names[0], []*data.ObjectInfo{})
require.Nil(t, actual)
})
@ -76,14 +76,14 @@ func TestTrimAfterObjectName(t *testing.T) {
func TestTrimAfterObjectID(t *testing.T) {
var (
objects []*api.ObjectInfo
objects []*data.ObjectInfo
ids []*object.ID
numberOfIDS = 3
)
for i := 0; i < numberOfIDS; i++ {
id := randID(t)
objects = append(objects, &api.ObjectInfo{ID: id})
objects = append(objects, &data.ObjectInfo{ID: id})
ids = append(ids, id)
}

11
api/layer/util.go
View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
)
@ -17,7 +18,7 @@ type (
// ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2.
ListObjectsInfo struct {
Prefixes []string
Objects []*api.ObjectInfo
Objects []*data.ObjectInfo
IsTruncated bool
}
@ -35,7 +36,7 @@ type (
// ObjectVersionInfo stores info about objects versions.
ObjectVersionInfo struct {
Object *api.ObjectInfo
Object *data.ObjectInfo
IsLatest bool
}
@ -65,11 +66,11 @@ func userHeaders(attrs []*object.Attribute) map[string]string {
return result
}
func objInfoFromMeta(bkt *api.BucketInfo, meta *object.Object) *api.ObjectInfo {
func objInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectInfo {
return objectInfoFromMeta(bkt, meta, "", "")
}
func objectInfoFromMeta(bkt *api.BucketInfo, meta *object.Object, prefix, delimiter string) *api.ObjectInfo {
func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object, prefix, delimiter string) *data.ObjectInfo {
var (
isDir bool
size int64
@ -110,7 +111,7 @@ func objectInfoFromMeta(bkt *api.BucketInfo, meta *object.Object, prefix, delimi
size = int64(meta.PayloadSize())
}
return &api.ObjectInfo{
return &data.ObjectInfo{
ID: meta.ID(),
CID: bkt.CID,
IsDir: isDir,

12
api/layer/util_test.go
View file

@ -9,7 +9,7 @@ import (
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/stretchr/testify/require"
)
@ -20,7 +20,7 @@ var (
defaultTestContentType = http.DetectContentType(defaultTestPayload)
)
func newTestObject(oid *object.ID, bkt *api.BucketInfo, name string) *object.Object {
func newTestObject(oid *object.ID, bkt *data.BucketInfo, name string) *object.Object {
filename := object.NewAttribute()
filename.SetKey(object.AttributeFileName)
filename.SetValue(name)
@ -44,8 +44,8 @@ func newTestObject(oid *object.ID, bkt *api.BucketInfo, name string) *object.Obj
return raw.Object()
}
func newTestInfo(oid *object.ID, bkt *api.BucketInfo, name string, isDir bool) *api.ObjectInfo {
info := &api.ObjectInfo{
func newTestInfo(oid *object.ID, bkt *data.BucketInfo, name string, isDir bool) *data.ObjectInfo {
info := &data.ObjectInfo{
ID: oid,
Name: name,
Bucket: bkt.Name,
@ -72,7 +72,7 @@ func Test_objectInfoFromMeta(t *testing.T) {
oid := object.NewID()
containerID := cid.New()
bkt := &api.BucketInfo{
bkt := &data.BucketInfo{
Name: "test-container",
CID: containerID,
Owner: uid,
@ -82,7 +82,7 @@ func Test_objectInfoFromMeta(t *testing.T) {
cases := []struct {
name string
prefix string
result *api.ObjectInfo
result *data.ObjectInfo
object *object.Object
delimiter string
}{

26
api/layer/versioning.go
View file

@ -8,13 +8,13 @@ import (
"strings"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
)
type objectVersions struct {
name string
objects []*api.ObjectInfo
objects []*data.ObjectInfo
addList []string
delList []string
isSorted bool
@ -40,7 +40,7 @@ func (v *objectVersions) isAddListEmpty() bool {
return len(v.addList) == 0
}
func (v *objectVersions) appendVersion(oi *api.ObjectInfo) {
func (v *objectVersions) appendVersion(oi *data.ObjectInfo) {
delVers := splitVersions(oi.Headers[versionsDelAttr])
v.objects = append(v.objects, oi)
@ -100,7 +100,7 @@ func (v *objectVersions) formAddList() {
}
}
func containsVersions(obj *api.ObjectInfo, versions []string) bool {
func containsVersions(obj *data.ObjectInfo, versions []string) bool {
header := obj.Headers[versionsAddAttr]
for _, version := range versions {
if !strings.Contains(header, version) {
@ -156,7 +156,7 @@ LOOP:
return commonAddedVersions, prevVersions, currentVersions
}
func (v *objectVersions) getLast() *api.ObjectInfo {
func (v *objectVersions) getLast() *data.ObjectInfo {
if v == nil || len(v.objects) == 0 {
return nil
}
@ -189,14 +189,14 @@ func (v *objectVersions) existedVersions() []string {
return res
}
func (v *objectVersions) getFiltered(reverse bool) []*api.ObjectInfo {
func (v *objectVersions) getFiltered(reverse bool) []*data.ObjectInfo {
if len(v.objects) == 0 {
return nil
}
v.sort()
existedVersions := v.existedVersions()
res := make([]*api.ObjectInfo, 0, len(v.objects))
res := make([]*data.ObjectInfo, 0, len(v.objects))
for _, version := range v.objects {
delMark := version.Headers[versionsDeleteMarkAttr]
@ -223,7 +223,7 @@ func (v *objectVersions) getDelHeader() string {
return strings.Join(v.delList, ",")
}
func (v *objectVersions) getVersion(oid *object.ID) *api.ObjectInfo {
func (v *objectVersions) getVersion(oid *object.ID) *data.ObjectInfo {
for _, version := range v.objects {
if version.Version() == oid.String() {
if contains(v.delList, oid.String()) {
@ -234,7 +234,7 @@ func (v *objectVersions) getVersion(oid *object.ID) *api.ObjectInfo {
}
return nil
}
func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*api.ObjectInfo, error) {
func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*data.ObjectInfo, error) {
bktInfo, err := n.GetBucketInfo(ctx, p.Bucket)
if err != nil {
return nil, err
@ -264,7 +264,7 @@ func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*Bu
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
var (
versions map[string]*objectVersions
allObjects = make([]*api.ObjectInfo, 0, p.MaxKeys)
allObjects = make([]*data.ObjectInfo, 0, p.MaxKeys)
res = &ListObjectVersionsInfo{}
reverse = true
)
@ -347,7 +347,7 @@ func contains(list []string, elem string) bool {
return false
}
func (n *layer) getBucketSettings(ctx context.Context, bktInfo *api.BucketInfo) (*BucketSettings, error) {
func (n *layer) getBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*BucketSettings, error) {
objInfo, err := n.getSystemObject(ctx, bktInfo, bktInfo.SettingsObjectName())
if err != nil {
return nil, err
@ -356,7 +356,7 @@ func (n *layer) getBucketSettings(ctx context.Context, bktInfo *api.BucketInfo)
return objectInfoToBucketSettings(objInfo), nil
}
func objectInfoToBucketSettings(info *api.ObjectInfo) *BucketSettings {
func objectInfoToBucketSettings(info *data.ObjectInfo) *BucketSettings {
res := &BucketSettings{}
enabled, ok := info.Headers[attrSettingsVersioningEnabled]
@ -368,7 +368,7 @@ func objectInfoToBucketSettings(info *api.ObjectInfo) *BucketSettings {
return res
}
func (n *layer) checkVersionsExist(ctx context.Context, bkt *api.BucketInfo, obj *VersionedObject) (*api.ObjectInfo, error) {
func (n *layer) checkVersionsExist(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.ObjectInfo, error) {
id := object.NewID()
if err := id.Parse(obj.VersionID); err != nil {
return nil, errors.GetAPIError(errors.ErrInvalidVersion)

67
api/layer/versioning_test.go
View file

@ -21,7 +21,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/session"
"github.com/nspcc-dev/neofs-api-go/pkg/token"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/logger"
"github.com/nspcc-dev/neofs-sdk-go/pool"
@ -208,7 +208,7 @@ func (t *testPool) WaitForContainerPresence(ctx context.Context, id *cid.ID, par
return nil
}
func (tc *testContext) putObject(content []byte) *api.ObjectInfo {
func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
Bucket: tc.bkt,
Object: tc.obj,
@ -221,7 +221,7 @@ func (tc *testContext) putObject(content []byte) *api.ObjectInfo {
return objInfo
}
func (tc *testContext) getObject(objectName, versionID string, needError bool) (*api.ObjectInfo, []byte) {
func (tc *testContext) getObject(objectName, versionID string, needError bool) (*data.ObjectInfo, []byte) {
objInfo, err := tc.layer.GetObjectInfo(tc.ctx, &HeadObjectParams{
Bucket: tc.bkt,
Object: objectName,
@ -254,7 +254,7 @@ func (tc *testContext) deleteObject(objectName, versionID string) {
}
}
func (tc *testContext) listObjectsV1() []*api.ObjectInfo {
func (tc *testContext) listObjectsV1() []*data.ObjectInfo {
res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{
ListObjectsParamsCommon: ListObjectsParamsCommon{
Bucket: tc.bkt,
@ -265,7 +265,7 @@ func (tc *testContext) listObjectsV1() []*api.ObjectInfo {
return res.Objects
}
func (tc *testContext) listObjectsV2() []*api.ObjectInfo {
func (tc *testContext) listObjectsV2() []*data.ObjectInfo {
res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{
ListObjectsParamsCommon: ListObjectsParamsCommon{
Bucket: tc.bkt,
@ -330,12 +330,7 @@ func prepareContext(t *testing.T) *testContext {
return &testContext{
ctx: ctx,
layer: NewLayer(l, tp, &CacheConfig{
Size: cache.DefaultObjectsCacheSize,
Lifetime: cache.DefaultObjectsCacheLifetime,
ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime,
ListObjectsSize: cache.DefaultObjectsListCacheSize},
),
layer: NewLayer(l, tp, DefaultCachesConfigs()),
bkt: bktName,
bktID: bktID,
obj: "obj1",
@ -457,7 +452,7 @@ func TestGetLastVersion(t *testing.T) {
for _, tc := range []struct {
versions *objectVersions
expected *api.ObjectInfo
expected *data.ObjectInfo
}{
{
versions: &objectVersions{},
@ -465,21 +460,21 @@ func TestGetLastVersion(t *testing.T) {
},
{
versions: &objectVersions{
objects: []*api.ObjectInfo{obj2, obj1},
objects: []*data.ObjectInfo{obj2, obj1},
addList: []string{obj1.Version(), obj2.Version()},
},
expected: obj2,
},
{
versions: &objectVersions{
objects: []*api.ObjectInfo{obj2, obj1, obj3},
objects: []*data.ObjectInfo{obj2, obj1, obj3},
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
},
expected: nil,
},
{
versions: &objectVersions{
objects: []*api.ObjectInfo{obj2, obj1, obj4},
objects: []*data.ObjectInfo{obj2, obj1, obj4},
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
delList: []string{obj2.Version()},
},
@ -487,7 +482,7 @@ func TestGetLastVersion(t *testing.T) {
},
{
versions: &objectVersions{
objects: []*api.ObjectInfo{obj1, obj5},
objects: []*data.ObjectInfo{obj1, obj5},
addList: []string{obj1.Version(), obj5.Version()},
delList: []string{obj1.Version()},
},
@ -495,13 +490,13 @@ func TestGetLastVersion(t *testing.T) {
},
{
versions: &objectVersions{
objects: []*api.ObjectInfo{obj5},
objects: []*data.ObjectInfo{obj5},
},
expected: nil,
},
{
versions: &objectVersions{
objects: []*api.ObjectInfo{obj1, obj2, obj3, obj6},
objects: []*data.ObjectInfo{obj1, obj2, obj3, obj6},
addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj6.Version()},
delList: []string{obj3.Version()},
},
@ -509,7 +504,7 @@ func TestGetLastVersion(t *testing.T) {
},
{
versions: &objectVersions{
objects: []*api.ObjectInfo{obj1, obj1V2},
objects: []*data.ObjectInfo{obj1, obj1V2},
addList: []string{obj1.Version(), obj1V2.Version()},
},
expected: obj1V2,
@ -530,51 +525,51 @@ func TestAppendVersions(t *testing.T) {
for _, tc := range []struct {
versions *objectVersions
objectToAdd *api.ObjectInfo
objectToAdd *data.ObjectInfo
expectedVersions *objectVersions
}{
{
versions: &objectVersions{},
objectToAdd: obj1,
expectedVersions: &objectVersions{
objects: []*api.ObjectInfo{obj1},
objects: []*data.ObjectInfo{obj1},
addList: []string{obj1.Version()},
isSorted: true,
},
},
{
versions: &objectVersions{objects: []*api.ObjectInfo{obj1}},
versions: &objectVersions{objects: []*data.ObjectInfo{obj1}},
objectToAdd: obj2,
expectedVersions: &objectVersions{
objects: []*api.ObjectInfo{obj1, obj2},
objects: []*data.ObjectInfo{obj1, obj2},
addList: []string{obj1.Version(), obj2.Version()},
isSorted: true,
},
},
{
versions: &objectVersions{objects: []*api.ObjectInfo{obj1, obj2}},
versions: &objectVersions{objects: []*data.ObjectInfo{obj1, obj2}},
objectToAdd: obj3,
expectedVersions: &objectVersions{
objects: []*api.ObjectInfo{obj1, obj2, obj3},
objects: []*data.ObjectInfo{obj1, obj2, obj3},
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
isSorted: true,
},
},
{
versions: &objectVersions{objects: []*api.ObjectInfo{obj1, obj2}},
versions: &objectVersions{objects: []*data.ObjectInfo{obj1, obj2}},
objectToAdd: obj4,
expectedVersions: &objectVersions{
objects: []*api.ObjectInfo{obj1, obj2, obj4},
objects: []*data.ObjectInfo{obj1, obj2, obj4},
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
delList: []string{obj2.Version()},
isSorted: true,
},
},
{
versions: &objectVersions{objects: []*api.ObjectInfo{obj5}},
versions: &objectVersions{objects: []*data.ObjectInfo{obj5}},
objectToAdd: obj6,
expectedVersions: &objectVersions{
objects: []*api.ObjectInfo{obj5, obj6},
objects: []*data.ObjectInfo{obj5, obj6},
addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj5.Version(), obj6.Version()},
isSorted: true,
},
@ -606,15 +601,15 @@ func TestSortAddHeaders(t *testing.T) {
expectedAddHeaders string
}{
{
versions: &objectVersions{objects: []*api.ObjectInfo{obj6, obj7, obj8}},
versions: &objectVersions{objects: []*data.ObjectInfo{obj6, obj7, obj8}},
expectedAddHeaders: joinVers(obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8),
},
{
versions: &objectVersions{objects: []*api.ObjectInfo{obj7, obj9}},
versions: &objectVersions{objects: []*data.ObjectInfo{obj7, obj9}},
expectedAddHeaders: joinVers(obj1, obj4, obj5, obj7, obj9),
},
{
versions: &objectVersions{objects: []*api.ObjectInfo{obj11, obj10, obj12}},
versions: &objectVersions{objects: []*data.ObjectInfo{obj11, obj10, obj12}},
expectedAddHeaders: joinVers(obj10, obj11, obj12),
},
} {
@ -622,7 +617,7 @@ func TestSortAddHeaders(t *testing.T) {
}
}
func joinVers(objs ...*api.ObjectInfo) string {
func joinVers(objs ...*data.ObjectInfo) string {
if len(objs) == 0 {
return ""
}
@ -643,7 +638,7 @@ func getOID(id byte) *object.ID {
return oid
}
func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *api.ObjectInfo {
func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo {
headers := make(map[string]string)
if addAttr != "" {
headers[versionsAddAttr] = addAttr
@ -655,14 +650,14 @@ func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *api.Objec
headers[versionsDeleteMarkAttr] = delMarkAttr
}
return &api.ObjectInfo{
return &data.ObjectInfo{
ID: getOID(id),
Name: strconv.Itoa(int(id)),
Headers: headers,
}
}
func getTestObjectInfoEpoch(epoch uint64, id byte, addAttr, delAttr, delMarkAttr string) *api.ObjectInfo {
func getTestObjectInfoEpoch(epoch uint64, id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo {
obj := getTestObjectInfo(id, addAttr, delAttr, delMarkAttr)
obj.CreationEpoch = epoch
return obj

5
authmate/authmate.go
View file

@ -22,6 +22,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-api-go/pkg/session"
"github.com/nspcc-dev/neofs-api-go/pkg/token"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
"github.com/nspcc-dev/neofs-sdk-go/policy"
@ -226,7 +227,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
}
address, err := tokens.
New(a.pool, secrets.EphemeralKey, tokens.DefaultCacheConfig()).
New(a.pool, secrets.EphemeralKey, cache.DefaultAccessBoxConfig()).
Put(ctx, cid, oid, box, lifetime.Exp, options.GatesPublicKeys...)
if err != nil {
return fmt.Errorf("failed to put bearer token: %w", err)
@ -268,7 +269,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
// ObtainSecret receives an existing secret access key from NeoFS and
// writes to io.Writer the secret access key.
func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error {
bearerCreds := tokens.New(a.pool, options.GatePrivateKey, tokens.DefaultCacheConfig())
bearerCreds := tokens.New(a.pool, options.GatePrivateKey, cache.DefaultAccessBoxConfig())
address := object.NewAddress()
if err := address.Parse(options.SecretAddress); err != nil {
return fmt.Errorf("failed to parse secret address: %w", err)

33
cmd/s3-gw/app.go
View file

@ -14,7 +14,6 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
"github.com/nspcc-dev/neofs-sdk-go/policy"
"github.com/nspcc-dev/neofs-sdk-go/pool"
@ -216,21 +215,25 @@ func (a *App) Server(ctx context.Context) {
close(a.webDone)
}
func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig {
cacheCfg := layer.CacheConfig{
ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime,
ListObjectsSize: cache.DefaultObjectsListCacheSize,
Size: cache.DefaultObjectsCacheSize,
Lifetime: cache.DefaultObjectsCacheLifetime,
}
func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CachesConfig {
cacheCfg := layer.DefaultCachesConfigs()
cacheCfg.Lifetime = getLifetime(v, l, cfgObjectsCacheLifetime, cacheCfg.Lifetime)
cacheCfg.Size = getSize(v, l, cfgObjectsCacheSize, cacheCfg.Size)
cacheCfg.Objects.Lifetime = getLifetime(v, l, cfgObjectsCacheLifetime, cacheCfg.Objects.Lifetime)
cacheCfg.Objects.Size = getSize(v, l, cfgObjectsCacheSize, cacheCfg.Objects.Size)
cacheCfg.ListObjectsLifetime = getLifetime(v, l, cfgListObjectsCacheLifetime, cacheCfg.ListObjectsLifetime)
cacheCfg.ListObjectsSize = getSize(v, l, cfgListObjectsCacheSize, cacheCfg.ListObjectsSize)
cacheCfg.ObjectsList.Lifetime = getLifetime(v, l, cfgListObjectsCacheLifetime, cacheCfg.ObjectsList.Lifetime)
cacheCfg.ObjectsList.Size = getSize(v, l, cfgListObjectsCacheSize, cacheCfg.ObjectsList.Size)
return &cacheCfg
cacheCfg.Buckets.Lifetime = getLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Buckets.Lifetime)
cacheCfg.Buckets.Size = getSize(v, l, cfgBucketsCacheSize, cacheCfg.Buckets.Size)
cacheCfg.Names.Lifetime = getLifetime(v, l, cfgNamesCacheLifetime, cacheCfg.Names.Lifetime)
cacheCfg.Names.Size = getSize(v, l, cfgNamesCacheSize, cacheCfg.Names.Size)
cacheCfg.System.Lifetime = getLifetime(v, l, cfgSystemLifetimeSize, cacheCfg.System.Lifetime)
cacheCfg.System.Size = getSize(v, l, cfgSystemCacheSize, cacheCfg.System.Size)
return cacheCfg
}
func getLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration {
@ -263,8 +266,8 @@ func getSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue int) i
return defaultValue
}
func getAccessBoxCacheConfig(v *viper.Viper, l *zap.Logger) *tokens.CacheConfig {
cacheCfg := tokens.DefaultCacheConfig()
func getAccessBoxCacheConfig(v *viper.Viper, l *zap.Logger) *cache.Config {
cacheCfg := cache.DefaultAccessBoxConfig()
cacheCfg.Lifetime = getLifetime(v, l, cfgAccessBoxCacheLifetime, cacheCfg.Lifetime)
cacheCfg.Size = getSize(v, l, cfgAccessBoxCacheSize, cacheCfg.Size)

6
cmd/s3-gw/app_settings.go
View file

@ -59,6 +59,12 @@ const ( // Settings.
cfgObjectsCacheSize = "cache.objects.size"
cfgListObjectsCacheLifetime = "cache.list.lifetime"
cfgListObjectsCacheSize = "cache.list.size"
cfgBucketsCacheLifetime = "cache.buckets.lifetime"
cfgBucketsCacheSize = "cache.buckets.size"
cfgNamesCacheLifetime = "cache.names.lifetime"
cfgNamesCacheSize = "cache.names.size"
cfgSystemLifetimeSize = "cache.system.lifetime"
cfgSystemCacheSize = "cache.system.size"
cfgAccessBoxCacheLifetime = "cache.accessbox.lifetime"
cfgAccessBoxCacheSize = "cache.accessbox.size"

7
creds/tokens/credentials.go
View file

@ -13,6 +13,7 @@ import (
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/pool"
)
@ -27,7 +28,7 @@ type (
cred struct {
key *keys.PrivateKey
pool pool.Pool
cache *AccessBoxCache
cache *cache.AccessBoxCache
}
)
@ -47,8 +48,8 @@ var bufferPool = sync.Pool{
var _ = New
// New creates new Credentials instance using given cli and key.
func New(conns pool.Pool, key *keys.PrivateKey, config *CacheConfig) Credentials {
return &cred{pool: conns, key: key, cache: NewAccessBoxCache(config)}
func New(conns pool.Pool, key *keys.PrivateKey, config *cache.Config) Credentials {
return &cred{pool: conns, key: key, cache: cache.NewAccessBoxCache(config)}
}
func (c *cred) acquireBuffer() *bytes.Buffer {

9
docs/configuration.md
View file

@ -114,6 +114,15 @@ cache:
list:
lifetime: 1m
size: 100
names:
lifetime: 1m
size: 1000
buckets:
lifetime: 1m
size: 500
system:
lifetime: 2m
size: 1000
accessbox:
lifetime: 5m
size: 10