forked from TrueCloudLab/frostfs-s3-gw
[#236] cache: Refactor ListObjectsCache
Replaced map in ListObjectsCache by gcache. Now ListObjectsCache keeps only objectIDs and requests ObjectInfo from cache or NeoFS. Refactored ListObjectsCache keys: removed delimiter and method fields. Now ListObjectsCache keeps cache with all objects versions. Signed-off-by: Angira Kekteeva <kira@nspcc.ru>
This commit is contained in:
parent
1bc2e51cbc
commit
1ece42b23f
8 changed files with 171 additions and 149 deletions
92
api/cache/objectslist.go
vendored
92
api/cache/objectslist.go
vendored
|
@ -1,11 +1,12 @@
|
||||||
package cache
|
package cache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sync"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/bluele/gcache"
|
||||||
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
)
|
)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -19,86 +20,75 @@ import (
|
||||||
the list of objects. Otherwise we send the request to NeoFS.
|
the list of objects. Otherwise we send the request to NeoFS.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// ObjectsListCache provides interface for cache of ListObjectsV2 in a layer struct.
|
|
||||||
type (
|
type (
|
||||||
|
// ObjectsListCache provides interface for cache of ListObjectsV2 in a layer struct.
|
||||||
ObjectsListCache interface {
|
ObjectsListCache interface {
|
||||||
Get(key ObjectsListKey) []*api.ObjectInfo
|
Get(key ObjectsListKey) []*object.ID
|
||||||
Put(key ObjectsListKey, objects []*api.ObjectInfo)
|
Put(key ObjectsListKey, oids []*object.ID) error
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
|
|
||||||
const DefaultObjectsListCacheLifetime = time.Second * 60
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// ListObjectsMethod is used to mark a cache entry for ListObjectsV1/V2.
|
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
|
||||||
ListObjectsMethod = "listObjects"
|
DefaultObjectsListCacheLifetime = time.Second * 60
|
||||||
// ListVersionsMethod is used to mark a cache entry for ListObjectVersions.
|
// DefaultObjectsListCacheSize is a default size of cache of ListObjects.
|
||||||
ListVersionsMethod = "listVersions"
|
DefaultObjectsListCacheSize = 1e5
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
// ListObjectsCache contains cache for ListObjects and ListObjectVersions.
|
// ListObjectsCache contains cache for ListObjects and ListObjectVersions.
|
||||||
ListObjectsCache struct {
|
ListObjectsCache struct {
|
||||||
cacheLifetime time.Duration
|
lifetime time.Duration
|
||||||
caches map[ObjectsListKey]objectsListEntry
|
cache gcache.Cache
|
||||||
mtx sync.RWMutex
|
|
||||||
}
|
|
||||||
objectsListEntry struct {
|
|
||||||
list []*api.ObjectInfo
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObjectsListKey is a key to find a ObjectsListCache's entry.
|
// ObjectsListKey is a key to find a ObjectsListCache's entry.
|
||||||
ObjectsListKey struct {
|
ObjectsListKey struct {
|
||||||
Method string
|
cid string
|
||||||
Key string
|
prefix string
|
||||||
Delimiter string
|
|
||||||
Prefix string
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewObjectsListCache is a constructor which creates an object of ListObjectsCache with given lifetime of entries.
|
// NewObjectsListCache is a constructor which creates an object of ListObjectsCache with given lifetime of entries.
|
||||||
func NewObjectsListCache(lifetime time.Duration) *ListObjectsCache {
|
func NewObjectsListCache(cacheSize int, lifetime time.Duration) *ListObjectsCache {
|
||||||
|
gc := gcache.New(cacheSize).LRU().Build()
|
||||||
|
|
||||||
return &ListObjectsCache{
|
return &ListObjectsCache{
|
||||||
caches: make(map[ObjectsListKey]objectsListEntry),
|
cache: gc,
|
||||||
cacheLifetime: lifetime,
|
lifetime: lifetime,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get return list of ObjectInfo.
|
// Get return list of ObjectInfo.
|
||||||
func (l *ListObjectsCache) Get(key ObjectsListKey) []*api.ObjectInfo {
|
func (l *ListObjectsCache) Get(key ObjectsListKey) []*object.ID {
|
||||||
l.mtx.RLock()
|
entry, err := l.cache.Get(key)
|
||||||
defer l.mtx.RUnlock()
|
if err != nil {
|
||||||
if val, ok := l.caches[key]; ok {
|
|
||||||
return val.list
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, ok := entry.([]*object.ID)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put put a list of objects to cache.
|
// Put puts a list of objects to cache.
|
||||||
func (l *ListObjectsCache) Put(key ObjectsListKey, objects []*api.ObjectInfo) {
|
func (l *ListObjectsCache) Put(key ObjectsListKey, oids []*object.ID) error {
|
||||||
if len(objects) == 0 {
|
if len(oids) == 0 {
|
||||||
return
|
return fmt.Errorf("list is empty, cid: %s, prefix: %s", key.cid, key.prefix)
|
||||||
}
|
}
|
||||||
var c objectsListEntry
|
|
||||||
l.mtx.Lock()
|
return l.cache.SetWithExpire(key, oids, l.lifetime)
|
||||||
defer l.mtx.Unlock()
|
|
||||||
c.list = objects
|
|
||||||
l.caches[key] = c
|
|
||||||
time.AfterFunc(l.cacheLifetime, func() {
|
|
||||||
l.mtx.Lock()
|
|
||||||
delete(l.caches, key)
|
|
||||||
l.mtx.Unlock()
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateObjectsListCacheKey returns ObjectsListKey with given CID, method, prefix, and delimiter.
|
// CreateObjectsListCacheKey returns ObjectsListKey with given CID, method, prefix, and delimiter.
|
||||||
func CreateObjectsListCacheKey(cid *cid.ID, method, prefix, delimiter string) (ObjectsListKey, error) {
|
func CreateObjectsListCacheKey(cid *cid.ID, prefix string) ObjectsListKey {
|
||||||
p := ObjectsListKey{
|
p := ObjectsListKey{
|
||||||
Method: method,
|
cid: cid.String(),
|
||||||
Key: cid.String(),
|
prefix: prefix,
|
||||||
Delimiter: delimiter,
|
|
||||||
Prefix: prefix,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return p, nil
|
return p
|
||||||
}
|
}
|
||||||
|
|
81
api/cache/objectslist_test.go
vendored
81
api/cache/objectslist_test.go
vendored
|
@ -3,16 +3,15 @@ package cache
|
||||||
import (
|
import (
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"sort"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
const testingCacheLifetime = 5 * time.Second
|
const testingCacheLifetime = 5 * time.Second
|
||||||
|
const testingCacheSize = 10
|
||||||
|
|
||||||
func randID(t *testing.T) *object.ID {
|
func randID(t *testing.T) *object.ID {
|
||||||
id := object.NewID()
|
id := object.NewID()
|
||||||
|
@ -31,83 +30,82 @@ func randSHA256Checksum(t *testing.T) (cs [sha256.Size]byte) {
|
||||||
func TestObjectsListCache(t *testing.T) {
|
func TestObjectsListCache(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
cacheSize = 10
|
cacheSize = 10
|
||||||
objects []*api.ObjectInfo
|
ids []*object.ID
|
||||||
userKey = "key"
|
userKey = "key"
|
||||||
)
|
)
|
||||||
|
|
||||||
for i := 0; i < cacheSize; i++ {
|
for i := 0; i < cacheSize; i++ {
|
||||||
id := randID(t)
|
id := randID(t)
|
||||||
objects = append(objects, &api.ObjectInfo{ID: id, Name: id.String()})
|
ids = append(ids, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(objects, func(i, j int) bool {
|
|
||||||
return objects[i].Name < objects[j].Name
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("lifetime", func(t *testing.T) {
|
t.Run("lifetime", func(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
cache = NewObjectsListCache(testingCacheLifetime)
|
cache = NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
||||||
cacheKey = ObjectsListKey{Key: userKey}
|
cacheKey = ObjectsListKey{cid: userKey}
|
||||||
)
|
)
|
||||||
|
|
||||||
cache.Put(cacheKey, objects)
|
err := cache.Put(cacheKey, ids)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
condition := func() bool {
|
condition := func() bool {
|
||||||
return cache.Get(cacheKey) == nil
|
return cache.Get(cacheKey) == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
require.Never(t, condition, cache.cacheLifetime, time.Second)
|
require.Never(t, condition, cache.lifetime, time.Second)
|
||||||
require.Eventually(t, condition, time.Second, 10*time.Millisecond)
|
require.Eventually(t, condition, time.Second, 10*time.Millisecond)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("get cache with empty delimiter, empty prefix", func(t *testing.T) {
|
t.Run("get cache with empty prefix", func(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
cache = NewObjectsListCache(testingCacheLifetime)
|
cache = NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
||||||
cacheKey = ObjectsListKey{Key: userKey}
|
cacheKey = ObjectsListKey{cid: userKey}
|
||||||
)
|
)
|
||||||
cache.Put(cacheKey, objects)
|
err := cache.Put(cacheKey, ids)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
actual := cache.Get(cacheKey)
|
actual := cache.Get(cacheKey)
|
||||||
|
|
||||||
require.Equal(t, len(objects), len(actual))
|
require.Equal(t, len(ids), len(actual))
|
||||||
for i := range objects {
|
for i := range ids {
|
||||||
require.Equal(t, objects[i], actual[i])
|
require.Equal(t, ids[i], actual[i])
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("get cache with delimiter and prefix", func(t *testing.T) {
|
t.Run("get cache with prefix", func(t *testing.T) {
|
||||||
cacheKey := ObjectsListKey{
|
cacheKey := ObjectsListKey{
|
||||||
Key: userKey,
|
cid: userKey,
|
||||||
Delimiter: "/",
|
prefix: "dir",
|
||||||
Prefix: "dir",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cache := NewObjectsListCache(testingCacheLifetime)
|
cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
||||||
cache.Put(cacheKey, objects)
|
err := cache.Put(cacheKey, ids)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
actual := cache.Get(cacheKey)
|
actual := cache.Get(cacheKey)
|
||||||
|
|
||||||
require.Equal(t, len(objects), len(actual))
|
require.Equal(t, len(ids), len(actual))
|
||||||
for i := range objects {
|
for i := range ids {
|
||||||
require.Equal(t, objects[i], actual[i])
|
require.Equal(t, ids[i], actual[i])
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("get cache with other delimiter and prefix", func(t *testing.T) {
|
t.Run("get cache with other prefix", func(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
cacheKey = ObjectsListKey{
|
cacheKey = ObjectsListKey{
|
||||||
Key: userKey,
|
cid: userKey,
|
||||||
Delimiter: "/",
|
prefix: "dir",
|
||||||
Prefix: "dir",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
newKey = ObjectsListKey{
|
newKey = ObjectsListKey{
|
||||||
Key: "key",
|
cid: "key",
|
||||||
Delimiter: "*",
|
prefix: "obj",
|
||||||
Prefix: "obj",
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
cache := NewObjectsListCache(testingCacheLifetime)
|
cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
||||||
cache.Put(cacheKey, objects)
|
err := cache.Put(cacheKey, ids)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
actual := cache.Get(newKey)
|
actual := cache.Get(newKey)
|
||||||
require.Nil(t, actual)
|
require.Nil(t, actual)
|
||||||
|
@ -116,15 +114,16 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
t.Run("get cache with non-existing key", func(t *testing.T) {
|
t.Run("get cache with non-existing key", func(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
cacheKey = ObjectsListKey{
|
cacheKey = ObjectsListKey{
|
||||||
Key: userKey,
|
cid: userKey,
|
||||||
}
|
}
|
||||||
newKey = ObjectsListKey{
|
newKey = ObjectsListKey{
|
||||||
Key: "asdf",
|
cid: "asdf",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
cache := NewObjectsListCache(testingCacheLifetime)
|
cache := NewObjectsListCache(testingCacheSize, testingCacheLifetime)
|
||||||
cache.Put(cacheKey, objects)
|
err := cache.Put(cacheKey, ids)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
actual := cache.Get(newKey)
|
actual := cache.Get(newKey)
|
||||||
require.Nil(t, actual)
|
require.Nil(t, actual)
|
||||||
|
|
|
@ -41,6 +41,7 @@ type (
|
||||||
Lifetime time.Duration
|
Lifetime time.Duration
|
||||||
Size int
|
Size int
|
||||||
ListObjectsLifetime time.Duration
|
ListObjectsLifetime time.Duration
|
||||||
|
ListObjectsSize int
|
||||||
}
|
}
|
||||||
|
|
||||||
// Params stores basic API parameters.
|
// Params stores basic API parameters.
|
||||||
|
@ -197,7 +198,7 @@ func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
|
||||||
return &layer{
|
return &layer{
|
||||||
pool: conns,
|
pool: conns,
|
||||||
log: log,
|
log: log,
|
||||||
listsCache: cache.NewObjectsListCache(config.ListObjectsLifetime),
|
listsCache: cache.NewObjectsListCache(config.ListObjectsSize, config.ListObjectsLifetime),
|
||||||
objCache: cache.New(config.Size, config.Lifetime),
|
objCache: cache.New(config.Size, config.Lifetime),
|
||||||
//todo reconsider cache params
|
//todo reconsider cache params
|
||||||
namesCache: cache.NewObjectsNameCache(1000, time.Minute),
|
namesCache: cache.NewObjectsNameCache(1000, time.Minute),
|
||||||
|
@ -671,7 +672,7 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
objects, err := n.listSortedObjectsFromNeoFS(ctx, allObjectParams{Bucket: bucketInfo})
|
objects, err := n.listSortedObjects(ctx, allObjectParams{Bucket: bucketInfo})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,7 @@ type (
|
||||||
attr string
|
attr string
|
||||||
val string
|
val string
|
||||||
cid *cid.ID
|
cid *cid.ID
|
||||||
|
prefix string
|
||||||
}
|
}
|
||||||
|
|
||||||
getParams struct {
|
getParams struct {
|
||||||
|
@ -82,6 +83,11 @@ func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]*object.ID,
|
||||||
opts.AddFilter(p.attr, filename, object.MatchStringEqual)
|
opts.AddFilter(p.attr, filename, object.MatchStringEqual)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if prefix, err := url.QueryUnescape(p.prefix); err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else if prefix != "" {
|
||||||
|
opts.AddFilter(object.AttributeFileName, prefix, object.MatchCommonPrefix)
|
||||||
|
}
|
||||||
return n.pool.SearchObject(ctx, new(client.SearchObjectParams).WithContainerID(p.cid).WithSearchFilters(opts), n.BearerOpt(ctx))
|
return n.pool.SearchObject(ctx, new(client.SearchObjectParams).WithContainerID(p.cid).WithSearchFilters(opts), n.BearerOpt(ctx))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -457,7 +463,7 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
|
||||||
return &result, nil
|
return &result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*api.ObjectInfo, error) {
|
func (n *layer) listSortedObjects(ctx context.Context, p allObjectParams) ([]*api.ObjectInfo, error) {
|
||||||
versions, err := n.getAllObjectsVersions(ctx, p.Bucket, p.Prefix, p.Delimiter)
|
versions, err := n.getAllObjectsVersions(ctx, p.Bucket, p.Prefix, p.Delimiter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -479,19 +485,34 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *api.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) {
|
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *api.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) {
|
||||||
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID})
|
var err error
|
||||||
|
|
||||||
|
cacheKey := cache.CreateObjectsListCacheKey(bkt.CID, prefix)
|
||||||
|
ids := n.listsCache.Get(cacheKey)
|
||||||
|
|
||||||
|
if ids == nil {
|
||||||
|
ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, prefix: prefix})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if err := n.listsCache.Put(cacheKey, ids); err != nil {
|
||||||
|
n.log.Error("couldn't cache list of objects", zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
versions := make(map[string]*objectVersions, len(ids)/2)
|
versions := make(map[string]*objectVersions, len(ids)/2)
|
||||||
for _, id := range ids {
|
|
||||||
meta, err := n.objectHead(ctx, bkt.CID, id)
|
for i := 0; i < len(ids); i++ {
|
||||||
if err != nil {
|
if ids[i] == nil {
|
||||||
n.log.Warn("could not fetch object meta", zap.Error(err))
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if oi := objectInfoFromMeta(bkt, meta, prefix, delimiter); oi != nil {
|
obj := n.objectFromObjectsCacheOrNeoFS(ctx, bkt.CID, ids[i])
|
||||||
|
if obj == nil {
|
||||||
|
// mark ids[i] as an address of invalid object
|
||||||
|
ids[i] = nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if oi := objectInfoFromMeta(bkt, obj, prefix, delimiter); oi != nil {
|
||||||
if isSystem(oi) {
|
if isSystem(oi) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -573,7 +594,6 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
bkt *api.BucketInfo
|
bkt *api.BucketInfo
|
||||||
cacheKey cache.ObjectsListKey
|
|
||||||
allObjects []*api.ObjectInfo
|
allObjects []*api.ObjectInfo
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -581,14 +601,7 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if cacheKey, err = cache.CreateObjectsListCacheKey(bkt.CID, cache.ListObjectsMethod, p.Prefix, p.Delimiter); err != nil {
|
allObjects, err = n.listSortedObjects(ctx, allObjectParams{
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
allObjects = n.listsCache.Get(cacheKey)
|
|
||||||
|
|
||||||
if allObjects == nil {
|
|
||||||
allObjects, err = n.listSortedObjectsFromNeoFS(ctx, allObjectParams{
|
|
||||||
Bucket: bkt,
|
Bucket: bkt,
|
||||||
Prefix: p.Prefix,
|
Prefix: p.Prefix,
|
||||||
Delimiter: p.Delimiter,
|
Delimiter: p.Delimiter,
|
||||||
|
@ -597,10 +610,6 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// putting to cache a copy of allObjects because allObjects can be modified further
|
|
||||||
n.listsCache.Put(cacheKey, append([]*api.ObjectInfo(nil), allObjects...))
|
|
||||||
}
|
|
||||||
|
|
||||||
return allObjects, nil
|
return allObjects, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -613,3 +622,22 @@ func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *api.BucketInfo
|
||||||
|
|
||||||
return settings.VersioningEnabled
|
return settings.VersioningEnabled
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (n *layer) objectFromObjectsCacheOrNeoFS(ctx context.Context, cid *cid.ID, oid *object.ID) *object.Object {
|
||||||
|
var (
|
||||||
|
err error
|
||||||
|
meta = n.objCache.Get(newAddress(cid, oid))
|
||||||
|
)
|
||||||
|
if meta == nil {
|
||||||
|
meta, err = n.objectHead(ctx, cid, oid)
|
||||||
|
if err != nil {
|
||||||
|
n.log.Warn("could not fetch object meta", zap.Error(err))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err = n.objCache.Put(*meta); err != nil {
|
||||||
|
n.log.Error("couldn't cache an object", zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return meta
|
||||||
|
}
|
||||||
|
|
|
@ -8,7 +8,6 @@ import (
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -146,23 +145,18 @@ func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*Bu
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||||
var versions map[string]*objectVersions
|
var (
|
||||||
res := &ListObjectVersionsInfo{}
|
versions map[string]*objectVersions
|
||||||
|
allObjects = make([]*api.ObjectInfo, 0, p.MaxKeys)
|
||||||
|
res = &ListObjectVersionsInfo{}
|
||||||
|
)
|
||||||
|
|
||||||
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
|
bkt, err := n.GetBucketInfo(ctx, p.Bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
cacheKey, err := cache.CreateObjectsListCacheKey(bkt.CID, cache.ListVersionsMethod, p.Prefix, p.Delimiter)
|
if versions, err = n.getAllObjectsVersions(ctx, bkt, p.Prefix, p.Delimiter); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
allObjects := n.listsCache.Get(cacheKey)
|
|
||||||
if allObjects == nil {
|
|
||||||
versions, err = n.getAllObjectsVersions(ctx, bkt, p.Prefix, p.Delimiter)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -172,15 +166,10 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar
|
||||||
}
|
}
|
||||||
sort.Strings(sortedNames)
|
sort.Strings(sortedNames)
|
||||||
|
|
||||||
allObjects = make([]*api.ObjectInfo, 0, p.MaxKeys)
|
|
||||||
for _, name := range sortedNames {
|
for _, name := range sortedNames {
|
||||||
allObjects = append(allObjects, versions[name].getFiltered()...)
|
allObjects = append(allObjects, versions[name].getFiltered()...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// putting to cache a copy of allObjects because allObjects can be modified further
|
|
||||||
n.listsCache.Put(cacheKey, append([]*api.ObjectInfo(nil), allObjects...))
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, obj := range allObjects {
|
for i, obj := range allObjects {
|
||||||
if obj.Name >= p.KeyMarker && obj.Version() >= p.VersionIDMarker {
|
if obj.Name >= p.KeyMarker && obj.Version() >= p.VersionIDMarker {
|
||||||
allObjects = allObjects[i:]
|
allObjects = allObjects[i:]
|
||||||
|
|
|
@ -331,7 +331,8 @@ func prepareContext(t *testing.T) *testContext {
|
||||||
layer: NewLayer(l, tp, &CacheConfig{
|
layer: NewLayer(l, tp, &CacheConfig{
|
||||||
Size: cache.DefaultObjectsCacheSize,
|
Size: cache.DefaultObjectsCacheSize,
|
||||||
Lifetime: cache.DefaultObjectsCacheLifetime,
|
Lifetime: cache.DefaultObjectsCacheLifetime,
|
||||||
ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime},
|
ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime,
|
||||||
|
ListObjectsSize: cache.DefaultObjectsListCacheSize},
|
||||||
),
|
),
|
||||||
bkt: bktName,
|
bkt: bktName,
|
||||||
bktID: bktID,
|
bktID: bktID,
|
||||||
|
|
|
@ -217,6 +217,7 @@ func (a *App) Server(ctx context.Context) {
|
||||||
func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig {
|
func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig {
|
||||||
cacheCfg := layer.CacheConfig{
|
cacheCfg := layer.CacheConfig{
|
||||||
ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime,
|
ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime,
|
||||||
|
ListObjectsSize: cache.DefaultObjectsListCacheSize,
|
||||||
Size: cache.DefaultObjectsCacheSize,
|
Size: cache.DefaultObjectsCacheSize,
|
||||||
Lifetime: cache.DefaultObjectsCacheLifetime,
|
Lifetime: cache.DefaultObjectsCacheLifetime,
|
||||||
}
|
}
|
||||||
|
@ -253,6 +254,18 @@ func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig {
|
||||||
cacheCfg.ListObjectsLifetime = lifetime
|
cacheCfg.ListObjectsLifetime = lifetime
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if v.IsSet(cfgListObjectsCacheSize) {
|
||||||
|
size := v.GetInt(cfgListObjectsCacheSize)
|
||||||
|
if size <= 0 {
|
||||||
|
l.Error("invalid cache size, using default value",
|
||||||
|
zap.Int("value in config", size),
|
||||||
|
zap.Int("default", cacheCfg.ListObjectsSize))
|
||||||
|
} else {
|
||||||
|
cacheCfg.Size = size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return &cacheCfg
|
return &cacheCfg
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -58,6 +58,7 @@ const ( // Settings.
|
||||||
cfgObjectsCacheLifetime = "cache.lifetime"
|
cfgObjectsCacheLifetime = "cache.lifetime"
|
||||||
cfgCacheSize = "cache.size"
|
cfgCacheSize = "cache.size"
|
||||||
cfgListObjectsCacheLifetime = "cache.list_objects_lifetime"
|
cfgListObjectsCacheLifetime = "cache.list_objects_lifetime"
|
||||||
|
cfgListObjectsCacheSize = "cache.list_objects_size"
|
||||||
|
|
||||||
// Policy.
|
// Policy.
|
||||||
cfgDefaultPolicy = "default_policy"
|
cfgDefaultPolicy = "default_policy"
|
||||||
|
|
Loading…
Reference in a new issue