[#236] api: Refactor caches: ObjectsList, Objects

Move ObjectsList from layer to cache package
Rename object_cache.go to objects.go

Signed-off-by: Angira Kekteeva <kira@nspcc.ru>

api/cache/objectslist.go

@@ -0,0 +1,104 @@
+package cache
+
+import (
+	"sync"
+	"time"
+
+	cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
+	"github.com/nspcc-dev/neofs-s3-gw/api"
+)
+
+/*
+	This is an implementation of a cache for ListObjectsV2/V1 which we can return to users when we receive a ListObjects
+	request.
+
+	The cache is a map which has a key: ObjectsListKey struct and a value: list of objects. After putting a record we
+	start a timer (via time.AfterFunc) that removes the record after DefaultObjectsListCacheLifetime value.
+
+	When we get a request from the user we just try to find the suitable and non-expired cache and then we return
+	the list of objects. Otherwise we send the request to NeoFS.
+*/
+
+// ObjectsListCache provides interface for cache of ListObjectsV2 in a layer struct.
+type (
+	ObjectsListCache interface {
+		Get(key ObjectsListKey) []*api.ObjectInfo
+		Put(key ObjectsListKey, objects []*api.ObjectInfo)
+	}
+)
+
+// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
+const DefaultObjectsListCacheLifetime = time.Second * 60
+
+const (
+	// ListObjectsMethod is used to mark a cache entry for ListObjectsV1/V2.
+	ListObjectsMethod = "listObjects"
+	// ListVersionsMethod is used to mark a cache entry for ListObjectVersions.
+	ListVersionsMethod = "listVersions"
+)
+
+type (
+	// ListObjectsCache contains cache for ListObjects and ListObjectVersions.
+	ListObjectsCache struct {
+		cacheLifetime time.Duration
+		caches        map[ObjectsListKey]objectsListEntry
+		mtx           sync.RWMutex
+	}
+	objectsListEntry struct {
+		list []*api.ObjectInfo
+	}
+	// ObjectsListKey is a key to find a ObjectsListCache's entry.
+	ObjectsListKey struct {
+		Method    string
+		Key       string
+		Delimiter string
+		Prefix    string
+	}
+)
+
+// NewObjectsListCache is a constructor which creates an object of ListObjectsCache with given lifetime of entries.
+func NewObjectsListCache(lifetime time.Duration) *ListObjectsCache {
+	return &ListObjectsCache{
+		caches:        make(map[ObjectsListKey]objectsListEntry),
+		cacheLifetime: lifetime,
+	}
+}
+
+// Get return list of ObjectInfo.
+func (l *ListObjectsCache) Get(key ObjectsListKey) []*api.ObjectInfo {
+	l.mtx.RLock()
+	defer l.mtx.RUnlock()
+	if val, ok := l.caches[key]; ok {
+		return val.list
+	}
+	return nil
+}
+
+// Put put a list of objects to cache.
+func (l *ListObjectsCache) Put(key ObjectsListKey, objects []*api.ObjectInfo) {
+	if len(objects) == 0 {
+		return
+	}
+	var c objectsListEntry
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
+	c.list = objects
+	l.caches[key] = c
+	time.AfterFunc(l.cacheLifetime, func() {
+		l.mtx.Lock()
+		delete(l.caches, key)
+		l.mtx.Unlock()
+	})
+}
+
+// CreateObjectsListCacheKey returns ObjectsListKey with given CID, method, prefix, and delimiter.
+func CreateObjectsListCacheKey(cid *cid.ID, method, prefix, delimiter string) (ObjectsListKey, error) {
+	p := ObjectsListKey{
+		Method:    method,
+		Key:       cid.String(),
+		Delimiter: delimiter,
+		Prefix:    prefix,
+	}
+
+	return p, nil
+}

api/cache/objectslist_test.go

@@ -0,0 +1,132 @@
+package cache
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/nspcc-dev/neofs-api-go/pkg/object"
+	"github.com/nspcc-dev/neofs-s3-gw/api"
+	"github.com/stretchr/testify/require"
+)
+
+const testingCacheLifetime = 5 * time.Second
+
+func randID(t *testing.T) *object.ID {
+	id := object.NewID()
+	id.SetSHA256(randSHA256Checksum(t))
+
+	return id
+}
+
+func randSHA256Checksum(t *testing.T) (cs [sha256.Size]byte) {
+	_, err := rand.Read(cs[:])
+	require.NoError(t, err)
+
+	return
+}
+
+func TestObjectsListCache(t *testing.T) {
+	var (
+		cacheSize = 10
+		objects   []*api.ObjectInfo
+		userKey   = "key"
+	)
+
+	for i := 0; i < cacheSize; i++ {
+		id := randID(t)
+		objects = append(objects, &api.ObjectInfo{ID: id, Name: id.String()})
+	}
+
+	sort.Slice(objects, func(i, j int) bool {
+		return objects[i].Name < objects[j].Name
+	})
+
+	t.Run("lifetime", func(t *testing.T) {
+		var (
+			cache    = NewObjectsListCache(testingCacheLifetime)
+			cacheKey = ObjectsListKey{Key: userKey}
+		)
+
+		cache.Put(cacheKey, objects)
+
+		condition := func() bool {
+			return cache.Get(cacheKey) == nil
+		}
+
+		require.Never(t, condition, cache.cacheLifetime, time.Second)
+		require.Eventually(t, condition, time.Second, 10*time.Millisecond)
+	})
+
+	t.Run("get cache with empty delimiter, empty prefix", func(t *testing.T) {
+		var (
+			cache    = NewObjectsListCache(testingCacheLifetime)
+			cacheKey = ObjectsListKey{Key: userKey}
+		)
+		cache.Put(cacheKey, objects)
+		actual := cache.Get(cacheKey)
+
+		require.Equal(t, len(objects), len(actual))
+		for i := range objects {
+			require.Equal(t, objects[i], actual[i])
+		}
+	})
+
+	t.Run("get cache with delimiter and prefix", func(t *testing.T) {
+		cacheKey := ObjectsListKey{
+			Key:       userKey,
+			Delimiter: "/",
+			Prefix:    "dir",
+		}
+
+		cache := NewObjectsListCache(testingCacheLifetime)
+		cache.Put(cacheKey, objects)
+		actual := cache.Get(cacheKey)
+
+		require.Equal(t, len(objects), len(actual))
+		for i := range objects {
+			require.Equal(t, objects[i], actual[i])
+		}
+	})
+
+	t.Run("get cache with other delimiter and prefix", func(t *testing.T) {
+		var (
+			cacheKey = ObjectsListKey{
+				Key:       userKey,
+				Delimiter: "/",
+				Prefix:    "dir",
+			}
+
+			newKey = ObjectsListKey{
+				Key:       "key",
+				Delimiter: "*",
+				Prefix:    "obj",
+			}
+		)
+
+		cache := NewObjectsListCache(testingCacheLifetime)
+		cache.Put(cacheKey, objects)
+
+		actual := cache.Get(newKey)
+		require.Nil(t, actual)
+	})
+
+	t.Run("get cache with non-existing key", func(t *testing.T) {
+		var (
+			cacheKey = ObjectsListKey{
+				Key: userKey,
+			}
+			newKey = ObjectsListKey{
+				Key: "asdf",
+			}
+		)
+
+		cache := NewObjectsListCache(testingCacheLifetime)
+		cache.Put(cacheKey, objects)
+
+		actual := cache.Get(newKey)
+		require.Nil(t, actual)
+	})
+}

api/layer/layer.go

@@ -29,7 +29,7 @@ type (
 	layer struct {
 		pool        pool.Pool
 		log         *zap.Logger
-		listsCache  ObjectsListCache
+		listsCache  cache.ObjectsListCache
 		objCache    cache.ObjectsCache
 		namesCache  cache.ObjectsNameCache
 		bucketCache cache.BucketCache
@@ -197,7 +197,7 @@ func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
 	return &layer{
 		pool:       conns,
 		log:        log,
-		listsCache: newListObjectsCache(config.ListObjectsLifetime),
+		listsCache: cache.NewObjectsListCache(config.ListObjectsLifetime),
 		objCache:   cache.New(config.Size, config.Lifetime),
 		//todo reconsider cache params
 		namesCache:  cache.NewObjectsNameCache(1000, time.Minute),

api/layer/object.go

@@ -15,6 +15,7 @@ import (
 	"github.com/nspcc-dev/neofs-api-go/pkg/object"
 	"github.com/nspcc-dev/neofs-api-go/pkg/owner"
 	"github.com/nspcc-dev/neofs-s3-gw/api"
+	"github.com/nspcc-dev/neofs-s3-gw/api/cache"
 	apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
 	"go.uber.org/zap"
 )
@@ -572,7 +573,7 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
 	var (
 		err        error
 		bkt        *api.BucketInfo
-		cacheKey   cacheOptions
+		cacheKey   cache.ObjectsListKey
 		allObjects []*api.ObjectInfo
 	)
 
@@ -580,7 +581,7 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) (
 		return nil, err
 	}
 
-	if cacheKey, err = createKey(bkt.CID, listObjectsMethod, p.Prefix, p.Delimiter); err != nil {
+	if cacheKey, err = cache.CreateObjectsListCacheKey(bkt.CID, cache.ListObjectsMethod, p.Prefix, p.Delimiter); err != nil {
 		return nil, err
 	}
 

api/layer/object_list_cache.go

@@ -1,96 +0,0 @@
-package layer
-
-import (
-	"sync"
-	"time"
-
-	cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
-	"github.com/nspcc-dev/neofs-s3-gw/api"
-)
-
-/*
-	This is an implementation of a cache for ListObjectsV2/V1 which we can return to users when we receive a ListObjects
-	request.
-
-	The cache is a map which has a key: cacheOptions struct and a value: list of objects. After putting a record we
-	start a timer (via time.AfterFunc) that removes the record after DefaultObjectsListCacheLifetime value.
-
-	When we get a request from the user we just try to find the suitable and non-expired cache and then we return
-	the list of objects. Otherwise we send the request to NeoFS.
-*/
-
-// ObjectsListCache provides interface for cache of ListObjectsV2 in a layer struct.
-type (
-	ObjectsListCache interface {
-		Get(key cacheOptions) []*api.ObjectInfo
-		Put(key cacheOptions, objects []*api.ObjectInfo)
-	}
-)
-
-// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
-const DefaultObjectsListCacheLifetime = time.Second * 60
-
-const (
-	listObjectsMethod  = "listObjects"
-	listVersionsMethod = "listVersions"
-)
-
-type (
-	listObjectsCache struct {
-		cacheLifetime time.Duration
-		caches        map[cacheOptions]cacheEntry
-		mtx           sync.RWMutex
-	}
-	cacheEntry struct {
-		list []*api.ObjectInfo
-	}
-	cacheOptions struct {
-		method    string
-		key       string
-		delimiter string
-		prefix    string
-	}
-)
-
-func newListObjectsCache(lifetime time.Duration) *listObjectsCache {
-	return &listObjectsCache{
-		caches:        make(map[cacheOptions]cacheEntry),
-		cacheLifetime: lifetime,
-	}
-}
-
-func (l *listObjectsCache) Get(key cacheOptions) []*api.ObjectInfo {
-	l.mtx.RLock()
-	defer l.mtx.RUnlock()
-	if val, ok := l.caches[key]; ok {
-		return val.list
-	}
-	return nil
-}
-
-func (l *listObjectsCache) Put(key cacheOptions, objects []*api.ObjectInfo) {
-	if len(objects) == 0 {
-		return
-	}
-	var c cacheEntry
-	l.mtx.Lock()
-	defer l.mtx.Unlock()
-	c.list = objects
-	l.caches[key] = c
-	time.AfterFunc(l.cacheLifetime, func() {
-		l.mtx.Lock()
-		delete(l.caches, key)
-		l.mtx.Unlock()
-	})
-}
-
-func createKey(cid *cid.ID, method, prefix, delimiter string) (cacheOptions, error) {
-	p := cacheOptions{
-		method:    method,
-		key:       cid.String(),
-		delimiter: delimiter,
-		prefix:    prefix,
-	}
-
-	return p, nil
-}

api/layer/object_test.go

@@ -3,17 +3,13 @@ package layer
 import (
 	"crypto/rand"
 	"crypto/sha256"
-	"sort"
 	"testing"
-	"time"
 
 	"github.com/nspcc-dev/neofs-api-go/pkg/object"
 	"github.com/nspcc-dev/neofs-s3-gw/api"
 	"github.com/stretchr/testify/require"
 )
 
-const testingCacheLifetime = 5 * time.Second
-
 func randID(t *testing.T) *object.ID {
 	id := object.NewID()
 	id.SetSHA256(randSHA256Checksum(t))
@@ -116,106 +112,3 @@ func TestTrimAfterObjectID(t *testing.T) {
 		require.Nil(t, actual)
 	})
 }
-
-func TestObjectsListCache(t *testing.T) {
-	var (
-		cacheSize = 10
-		objects   []*api.ObjectInfo
-		userKey   = "key"
-	)
-
-	for i := 0; i < cacheSize; i++ {
-		id := randID(t)
-		objects = append(objects, &api.ObjectInfo{ID: id, Name: id.String()})
-	}
-
-	sort.Slice(objects, func(i, j int) bool {
-		return objects[i].Name < objects[j].Name
-	})
-
-	t.Run("lifetime", func(t *testing.T) {
-		var (
-			cache    = newListObjectsCache(testingCacheLifetime)
-			cacheKey = cacheOptions{key: userKey}
-		)
-
-		cache.Put(cacheKey, objects)
-
-		condition := func() bool {
-			return cache.Get(cacheKey) == nil
-		}
-
-		require.Never(t, condition, cache.cacheLifetime, time.Second)
-		require.Eventually(t, condition, time.Second, 10*time.Millisecond)
-	})
-
-	t.Run("get cache with empty delimiter, empty prefix", func(t *testing.T) {
-		var (
-			cache    = newListObjectsCache(testingCacheLifetime)
-			cacheKey = cacheOptions{key: userKey}
-		)
-		cache.Put(cacheKey, objects)
-		actual := cache.Get(cacheKey)
-
-		require.Equal(t, len(objects), len(actual))
-		for i := range objects {
-			require.Equal(t, objects[i], actual[i])
-		}
-	})
-
-	t.Run("get cache with delimiter and prefix", func(t *testing.T) {
-		cacheKey := cacheOptions{
-			key:       userKey,
-			delimiter: "/",
-			prefix:    "dir",
-		}
-
-		cache := newListObjectsCache(testingCacheLifetime)
-		cache.Put(cacheKey, objects)
-		actual := cache.Get(cacheKey)
-
-		require.Equal(t, len(objects), len(actual))
-		for i := range objects {
-			require.Equal(t, objects[i], actual[i])
-		}
-	})
-
-	t.Run("get cache with other delimiter and prefix", func(t *testing.T) {
-		var (
-			cacheKey = cacheOptions{
-				key:       userKey,
-				delimiter: "/",
-				prefix:    "dir",
-			}
-
-			newKey = cacheOptions{
-				key:       "key",
-				delimiter: "*",
-				prefix:    "obj",
-			}
-		)
-
-		cache := newListObjectsCache(testingCacheLifetime)
-		cache.Put(cacheKey, objects)
-
-		actual := cache.Get(newKey)
-		require.Nil(t, actual)
-	})
-
-	t.Run("get cache with non-existing key", func(t *testing.T) {
-		var (
-			cacheKey = cacheOptions{
-				key: userKey,
-			}
-			newKey = cacheOptions{
-				key: "asdf",
-			}
-		)
-
-		cache := newListObjectsCache(testingCacheLifetime)
-		cache.Put(cacheKey, objects)
-
-		actual := cache.Get(newKey)
-		require.Nil(t, actual)
-	})
-}

api/layer/versioning.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/nspcc-dev/neofs-api-go/pkg/object"
 	"github.com/nspcc-dev/neofs-s3-gw/api"
+	"github.com/nspcc-dev/neofs-s3-gw/api/cache"
 	"github.com/nspcc-dev/neofs-s3-gw/api/errors"
 )
 
@@ -153,7 +154,7 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar
 		return nil, err
 	}
 
-	cacheKey, err := createKey(bkt.CID, listVersionsMethod, p.Prefix, p.Delimiter)
+	cacheKey, err := cache.CreateObjectsListCacheKey(bkt.CID, cache.ListVersionsMethod, p.Prefix, p.Delimiter)
 	if err != nil {
 		return nil, err
 	}

api/layer/versioning_test.go

@@ -331,7 +331,7 @@ func prepareContext(t *testing.T) *testContext {
 		layer: NewLayer(l, tp, &CacheConfig{
 			Size:                cache.DefaultObjectsCacheSize,
 			Lifetime:            cache.DefaultObjectsCacheLifetime,
-			ListObjectsLifetime: DefaultObjectsListCacheLifetime},
+			ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime},
 		),
 		bkt:      bktName,
 		bktID:    bktID,

cmd/s3-gw/app.go

@@ -216,7 +216,7 @@ func (a *App) Server(ctx context.Context) {
 
 func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig {
 	cacheCfg := layer.CacheConfig{
-		ListObjectsLifetime: layer.DefaultObjectsListCacheLifetime,
+		ListObjectsLifetime: cache.DefaultObjectsListCacheLifetime,
 		Size:                cache.DefaultObjectsCacheSize,
 		Lifetime:            cache.DefaultObjectsCacheLifetime,
 	}