forked from TrueCloudLab/frostfs-s3-gw
[#713] Fix cache access
User can access only keys that he puts into caches Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
parent
c6098e53f6
commit
3d31c2ab4a
14 changed files with 365 additions and 196 deletions
69
api/cache/access_control.go
vendored
Normal file
69
api/cache/access_control.go
vendored
Normal file
|
@ -0,0 +1,69 @@
|
||||||
|
package cache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/bluele/gcache"
|
||||||
|
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AccessControlCache provides lru cache for objects.
|
||||||
|
type AccessControlCache struct {
|
||||||
|
cache gcache.Cache
|
||||||
|
logger *zap.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// DefaultAccessControlCacheLifetime is a default lifetime of entries in access' cache.
|
||||||
|
DefaultAccessControlCacheLifetime = 1 * time.Minute
|
||||||
|
// DefaultAccessControlCacheSize is a default maximum number of entries in access' cache.
|
||||||
|
DefaultAccessControlCacheSize = 1e5
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultAccessControlConfig returns new default cache expiration values.
|
||||||
|
func DefaultAccessControlConfig(logger *zap.Logger) *Config {
|
||||||
|
return &Config{
|
||||||
|
Size: DefaultAccessControlCacheSize,
|
||||||
|
Lifetime: DefaultAccessControlCacheLifetime,
|
||||||
|
Logger: logger,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAccessControlCache creates an object of AccessControlCache.
|
||||||
|
func NewAccessControlCache(config *Config) *AccessControlCache {
|
||||||
|
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||||
|
return &AccessControlCache{cache: gc, logger: config.Logger}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns true if such key exists.
|
||||||
|
func (o *AccessControlCache) Get(owner user.ID, key string) bool {
|
||||||
|
entry, err := o.cache.Get(cacheKey(owner, key))
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
result, ok := entry.(bool)
|
||||||
|
if !ok {
|
||||||
|
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put puts an item to cache.
|
||||||
|
func (o *AccessControlCache) Put(owner user.ID, key string) error {
|
||||||
|
return o.cache.Set(cacheKey(owner, key), true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete deletes an object from cache.
|
||||||
|
func (o *AccessControlCache) Delete(owner user.ID, key string) bool {
|
||||||
|
return o.cache.Remove(cacheKey(owner, key))
|
||||||
|
}
|
||||||
|
|
||||||
|
func cacheKey(owner user.ID, key string) string {
|
||||||
|
return owner.EncodeToString() + key
|
||||||
|
}
|
5
api/cache/objectslist.go
vendored
5
api/cache/objectslist.go
vendored
|
@ -2,6 +2,7 @@ package cache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -56,6 +57,10 @@ func DefaultObjectsListConfig(logger *zap.Logger) *Config {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (k *ObjectsListKey) String() string {
|
||||||
|
return k.cid.EncodeToString() + k.prefix + strconv.FormatBool(k.latestOnly)
|
||||||
|
}
|
||||||
|
|
||||||
// NewObjectsListCache is a constructor which creates an object of ListObjectsCache with the given lifetime of entries.
|
// NewObjectsListCache is a constructor which creates an object of ListObjectsCache with the given lifetime of entries.
|
||||||
func NewObjectsListCache(config *Config) *ObjectsListCache {
|
func NewObjectsListCache(config *Config) *ObjectsListCache {
|
||||||
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||||
|
|
205
api/layer/cache.go
Normal file
205
api/layer/cache.go
Normal file
|
@ -0,0 +1,205 @@
|
||||||
|
package layer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
|
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||||
|
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||||
|
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Cache struct {
|
||||||
|
logger *zap.Logger
|
||||||
|
listsCache *cache.ObjectsListCache
|
||||||
|
objCache *cache.ObjectsCache
|
||||||
|
namesCache *cache.ObjectsNameCache
|
||||||
|
bucketCache *cache.BucketCache
|
||||||
|
systemCache *cache.SystemCache
|
||||||
|
accessCache *cache.AccessControlCache
|
||||||
|
}
|
||||||
|
|
||||||
|
// CachesConfig contains params for caches.
|
||||||
|
type CachesConfig struct {
|
||||||
|
Logger *zap.Logger
|
||||||
|
Objects *cache.Config
|
||||||
|
ObjectsList *cache.Config
|
||||||
|
Names *cache.Config
|
||||||
|
Buckets *cache.Config
|
||||||
|
System *cache.Config
|
||||||
|
AccessControl *cache.Config
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultCachesConfigs returns filled configs.
|
||||||
|
func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig {
|
||||||
|
return &CachesConfig{
|
||||||
|
Logger: logger,
|
||||||
|
Objects: cache.DefaultObjectsConfig(logger),
|
||||||
|
ObjectsList: cache.DefaultObjectsListConfig(logger),
|
||||||
|
Names: cache.DefaultObjectsNameConfig(logger),
|
||||||
|
Buckets: cache.DefaultBucketConfig(logger),
|
||||||
|
System: cache.DefaultSystemConfig(logger),
|
||||||
|
AccessControl: cache.DefaultAccessControlConfig(logger),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewCache(cfg *CachesConfig) *Cache {
|
||||||
|
return &Cache{
|
||||||
|
logger: cfg.Logger,
|
||||||
|
listsCache: cache.NewObjectsListCache(cfg.ObjectsList),
|
||||||
|
objCache: cache.New(cfg.Objects),
|
||||||
|
namesCache: cache.NewObjectsNameCache(cfg.Names),
|
||||||
|
bucketCache: cache.NewBucketCache(cfg.Buckets),
|
||||||
|
systemCache: cache.NewSystemCache(cfg.System),
|
||||||
|
accessCache: cache.NewAccessControlCache(cfg.AccessControl),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetBucket(name string) *data.BucketInfo {
|
||||||
|
return c.bucketCache.Get(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
|
||||||
|
if err := c.bucketCache.Put(bktInfo); err != nil {
|
||||||
|
c.logger.Warn("couldn't put bucket info into cache",
|
||||||
|
zap.String("bucket name", bktInfo.Name),
|
||||||
|
zap.Stringer("bucket cid", bktInfo.CID),
|
||||||
|
zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) DeleteBucket(name string) {
|
||||||
|
c.bucketCache.Delete(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) CleanListCacheEntriesContainingObject(objectName string, cnrID cid.ID) {
|
||||||
|
c.listsCache.CleanCacheEntriesContainingObject(objectName, cnrID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) DeleteObjectName(cnrID cid.ID, bktName, objName string) {
|
||||||
|
c.namesCache.Delete(bktName + "/" + objName)
|
||||||
|
c.listsCache.CleanCacheEntriesContainingObject(objName, cnrID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) DeleteObject(addr oid.Address) {
|
||||||
|
c.objCache.Delete(addr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetObject(owner user.ID, addr oid.Address) *data.ExtendedObjectInfo {
|
||||||
|
if !c.accessCache.Get(owner, addr.String()) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.objCache.GetObject(addr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetLastObject(owner user.ID, bktName, objName string) *data.ExtendedObjectInfo {
|
||||||
|
addr := c.namesCache.Get(bktName + "/" + objName)
|
||||||
|
if addr == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.GetObject(owner, *addr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
|
||||||
|
if err := c.objCache.PutObject(extObjInfo); err != nil {
|
||||||
|
c.logger.Warn("couldn't add object to cache", zap.Error(err),
|
||||||
|
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
|
||||||
|
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
|
||||||
|
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutObjectWithName(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
|
||||||
|
c.PutObject(owner, extObjInfo)
|
||||||
|
|
||||||
|
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
|
||||||
|
c.logger.Warn("couldn't put obj address to name cache",
|
||||||
|
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
|
||||||
|
zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetList(owner user.ID, key cache.ObjectsListKey) []*data.NodeVersion {
|
||||||
|
if !c.accessCache.Get(owner, key.String()) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.listsCache.GetVersions(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
|
||||||
|
if err := c.listsCache.PutVersions(key, list); err != nil {
|
||||||
|
c.logger.Warn("couldn't cache list of objects", zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
||||||
|
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetTagging(key string) map[string]string {
|
||||||
|
return c.systemCache.GetTagging(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutTagging(key string, tags map[string]string) {
|
||||||
|
if err := c.systemCache.PutTagging(key, tags); err != nil {
|
||||||
|
c.logger.Error("couldn't cache tags", zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) DeleteTagging(key string) {
|
||||||
|
c.systemCache.Delete(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetLockInfo(key string) *data.LockInfo {
|
||||||
|
return c.systemCache.GetLockInfo(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutLockInfo(key string, lockInfo *data.LockInfo) {
|
||||||
|
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
|
||||||
|
c.logger.Error("couldn't cache lock info", zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetSettings(bktInfo *data.BucketInfo) *data.BucketSettings {
|
||||||
|
key := bktInfo.Name + bktInfo.SettingsObjectName()
|
||||||
|
return c.systemCache.GetSettings(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutSettings(bktInfo *data.BucketInfo, settings *data.BucketSettings) {
|
||||||
|
key := bktInfo.Name + bktInfo.SettingsObjectName()
|
||||||
|
if err := c.systemCache.PutSettings(key, settings); err != nil {
|
||||||
|
c.logger.Warn("couldn't cache bucket settings", zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetCORS(bkt *data.BucketInfo) *data.CORSConfiguration {
|
||||||
|
return c.systemCache.GetCORS(bkt.Name + bkt.CORSObjectName())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutCORS(bkt *data.BucketInfo, settings *data.CORSConfiguration) {
|
||||||
|
if err := c.systemCache.PutCORS(bkt.Name+bkt.CORSObjectName(), settings); err != nil {
|
||||||
|
c.logger.Warn("couldn't cache cors", zap.String("bucket", bkt.Name), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) {
|
||||||
|
c.systemCache.Delete(bktInfo.Name + bktInfo.CORSObjectName())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetNotificationConfiguration(bktInfo *data.BucketInfo) *data.NotificationConfiguration {
|
||||||
|
key := bktInfo.Name + bktInfo.NotificationConfigurationObjectName()
|
||||||
|
return c.systemCache.GetNotificationConfiguration(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutNotificationConfiguration(bktInfo *data.BucketInfo, configuration *data.NotificationConfiguration) {
|
||||||
|
key := bktInfo.Name + bktInfo.NotificationConfigurationObjectName()
|
||||||
|
if err := c.systemCache.PutNotificationConfiguration(key, configuration); err != nil {
|
||||||
|
c.logger.Warn("couldn't cache notification configuration", zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
|
@ -6,7 +6,6 @@ import (
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
||||||
|
@ -15,8 +14,8 @@ func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectV
|
||||||
tags map[string]string
|
tags map[string]string
|
||||||
)
|
)
|
||||||
|
|
||||||
tags = n.systemCache.GetTagging(objectTaggingCacheKey(objVersion))
|
tags = n.cache.GetTagging(objectTaggingCacheKey(objVersion))
|
||||||
lockInfo := n.systemCache.GetLockInfo(lockObjectKey(objVersion))
|
lockInfo := n.cache.GetLockInfo(lockObjectKey(objVersion))
|
||||||
|
|
||||||
if tags != nil && lockInfo != nil {
|
if tags != nil && lockInfo != nil {
|
||||||
return tags, lockInfo, nil
|
return tags, lockInfo, nil
|
||||||
|
@ -37,13 +36,8 @@ func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectV
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutTagging(objectTaggingCacheKey(objVersion), tags); err != nil {
|
n.cache.PutTagging(objectTaggingCacheKey(objVersion), tags)
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
n.cache.PutLockInfo(lockObjectKey(objVersion), lockInfo)
|
||||||
}
|
|
||||||
|
|
||||||
if err = n.systemCache.PutLockInfo(lockObjectKey(objVersion), lockInfo); err != nil {
|
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return tags, lockInfo, nil
|
return tags, lockInfo, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -70,10 +70,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.bucketCache.Put(info); err != nil {
|
n.cache.PutBucket(info)
|
||||||
log.Warn("could not put bucket info into cache",
|
|
||||||
zap.String("bucket_name", info.Name), zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
@ -152,12 +149,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
||||||
return nil, fmt.Errorf("set container eacl: %w", err)
|
return nil, fmt.Errorf("set container eacl: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.bucketCache.Put(bktInfo); err != nil {
|
n.cache.PutBucket(bktInfo)
|
||||||
n.log.Warn("couldn't put bucket info into cache",
|
|
||||||
zap.String("bucket name", bktInfo.Name),
|
|
||||||
zap.Stringer("bucket cid", bktInfo.CID),
|
|
||||||
zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return bktInfo, nil
|
return bktInfo, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -64,15 +64,13 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutCORS(systemObjectKey(p.BktInfo, prm.Filepath), cors); err != nil {
|
n.cache.PutCORS(p.BktInfo, cors)
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error) {
|
func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error) {
|
||||||
cors, err := n.getCORS(ctx, bktInfo, bktInfo.CORSObjectName())
|
cors, err := n.getCORS(ctx, bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errorsStd.Is(err, ErrNodeNotFound) {
|
if errorsStd.Is(err, ErrNodeNotFound) {
|
||||||
return nil, errors.GetAPIError(errors.ErrNoSuchCORSConfiguration)
|
return nil, errors.GetAPIError(errors.ErrNoSuchCORSConfiguration)
|
||||||
|
@ -95,7 +93,7 @@ func (n *layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
n.systemCache.Delete(systemObjectKey(bktInfo, bktInfo.CORSObjectName()))
|
n.cache.DeleteCORS(bktInfo)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,7 +14,6 @@ import (
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
||||||
|
@ -51,11 +50,7 @@ type (
|
||||||
anonKey AnonymousKey
|
anonKey AnonymousKey
|
||||||
resolver BucketResolver
|
resolver BucketResolver
|
||||||
ncontroller EventListener
|
ncontroller EventListener
|
||||||
listsCache *cache.ObjectsListCache
|
cache *Cache
|
||||||
objCache *cache.ObjectsCache
|
|
||||||
namesCache *cache.ObjectsNameCache
|
|
||||||
bucketCache *cache.BucketCache
|
|
||||||
systemCache *cache.SystemCache
|
|
||||||
treeService TreeService
|
treeService TreeService
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -72,15 +67,6 @@ type (
|
||||||
Key *keys.PrivateKey
|
Key *keys.PrivateKey
|
||||||
}
|
}
|
||||||
|
|
||||||
// CachesConfig contains params for caches.
|
|
||||||
CachesConfig struct {
|
|
||||||
Objects *cache.Config
|
|
||||||
ObjectsList *cache.Config
|
|
||||||
Names *cache.Config
|
|
||||||
Buckets *cache.Config
|
|
||||||
System *cache.Config
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetObjectParams stores object get request parameters.
|
// GetObjectParams stores object get request parameters.
|
||||||
GetObjectParams struct {
|
GetObjectParams struct {
|
||||||
Range *RangeParams
|
Range *RangeParams
|
||||||
|
@ -280,17 +266,6 @@ func (f MsgHandlerFunc) HandleMessage(ctx context.Context, msg *nats.Msg) error
|
||||||
return f(ctx, msg)
|
return f(ctx, msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultCachesConfigs returns filled configs.
|
|
||||||
func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig {
|
|
||||||
return &CachesConfig{
|
|
||||||
Objects: cache.DefaultObjectsConfig(logger),
|
|
||||||
ObjectsList: cache.DefaultObjectsListConfig(logger),
|
|
||||||
Names: cache.DefaultObjectsNameConfig(logger),
|
|
||||||
Buckets: cache.DefaultBucketConfig(logger),
|
|
||||||
System: cache.DefaultSystemConfig(logger),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewLayer creates an instance of a layer. It checks credentials
|
// NewLayer creates an instance of a layer. It checks credentials
|
||||||
// and establishes gRPC connection with the node.
|
// and establishes gRPC connection with the node.
|
||||||
func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client {
|
func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client {
|
||||||
|
@ -299,11 +274,7 @@ func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client {
|
||||||
log: log,
|
log: log,
|
||||||
anonKey: config.AnonKey,
|
anonKey: config.AnonKey,
|
||||||
resolver: config.Resolver,
|
resolver: config.Resolver,
|
||||||
listsCache: cache.NewObjectsListCache(config.Caches.ObjectsList),
|
cache: NewCache(config.Caches),
|
||||||
objCache: cache.New(config.Caches.Objects),
|
|
||||||
namesCache: cache.NewObjectsNameCache(config.Caches.Names),
|
|
||||||
bucketCache: cache.NewBucketCache(config.Caches.Buckets),
|
|
||||||
systemCache: cache.NewSystemCache(config.Caches.System),
|
|
||||||
treeService: config.TreeService,
|
treeService: config.TreeService,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -365,7 +336,7 @@ func (n *layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInf
|
||||||
return nil, fmt.Errorf("unescape bucket name: %w", err)
|
return nil, fmt.Errorf("unescape bucket name: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if bktInfo := n.bucketCache.Get(name); bktInfo != nil {
|
if bktInfo := n.cache.GetBucket(name); bktInfo != nil {
|
||||||
return bktInfo, nil
|
return bktInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -561,7 +532,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
||||||
}
|
}
|
||||||
|
|
||||||
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID)
|
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID)
|
||||||
n.listsCache.CleanCacheEntriesContainingObject(obj.Name, bkt.CID)
|
n.cache.CleanListCacheEntriesContainingObject(obj.Name, bkt.CID)
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -604,8 +575,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|
||||||
n.namesCache.Delete(bkt.Name + "/" + obj.Name)
|
n.cache.DeleteObjectName(bkt.CID, bkt.Name, obj.Name)
|
||||||
n.listsCache.CleanCacheEntriesContainingObject(obj.Name, bkt.CID)
|
|
||||||
|
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
@ -681,6 +651,6 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
return errors.GetAPIError(errors.ErrBucketNotEmpty)
|
return errors.GetAPIError(errors.ErrBucketNotEmpty)
|
||||||
}
|
}
|
||||||
|
|
||||||
n.bucketCache.Delete(p.BktInfo.Name)
|
n.cache.DeleteBucket(p.BktInfo.Name)
|
||||||
return n.neoFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
return n.neoFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||||
}
|
}
|
||||||
|
|
|
@ -461,7 +461,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
zap.Error(err))
|
zap.Error(err))
|
||||||
}
|
}
|
||||||
addr.SetObject(partInfo.OID)
|
addr.SetObject(partInfo.OID)
|
||||||
n.objCache.Delete(addr)
|
n.cache.DeleteObject(addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
return uploadData, obj, n.treeService.DeleteMultipartUpload(ctx, p.Info.Bkt, multipartInfo.ID)
|
return uploadData, obj, n.treeService.DeleteMultipartUpload(ctx, p.Info.Bkt, multipartInfo.ID)
|
||||||
|
|
|
@ -25,13 +25,11 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
|
||||||
return fmt.Errorf("marshal notify configuration: %w", err)
|
return fmt.Errorf("marshal notify configuration: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sysName := p.BktInfo.NotificationConfigurationObjectName()
|
|
||||||
|
|
||||||
prm := PrmObjectCreate{
|
prm := PrmObjectCreate{
|
||||||
Container: p.BktInfo.CID,
|
Container: p.BktInfo.CID,
|
||||||
Creator: p.BktInfo.Owner,
|
Creator: p.BktInfo.Owner,
|
||||||
Payload: bytes.NewReader(confXML),
|
Payload: bytes.NewReader(confXML),
|
||||||
Filepath: sysName,
|
Filepath: p.BktInfo.NotificationConfigurationObjectName(),
|
||||||
CopiesNumber: p.CopiesNumber,
|
CopiesNumber: p.CopiesNumber,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -55,17 +53,13 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutNotificationConfiguration(systemObjectKey(p.BktInfo, sysName), p.Configuration); err != nil {
|
n.cache.PutNotificationConfiguration(p.BktInfo, p.Configuration)
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) GetBucketNotificationConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.NotificationConfiguration, error) {
|
func (n *layer) GetBucketNotificationConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.NotificationConfiguration, error) {
|
||||||
systemCacheKey := systemObjectKey(bktInfo, bktInfo.NotificationConfigurationObjectName())
|
if conf := n.cache.GetNotificationConfiguration(bktInfo); conf != nil {
|
||||||
|
|
||||||
if conf := n.systemCache.GetNotificationConfiguration(systemCacheKey); conf != nil {
|
|
||||||
return conf, nil
|
return conf, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -88,11 +82,7 @@ func (n *layer) GetBucketNotificationConfiguration(ctx context.Context, bktInfo
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutNotificationConfiguration(systemCacheKey, conf); err != nil {
|
n.cache.PutNotificationConfiguration(bktInfo, conf)
|
||||||
n.log.Warn("couldn't put system meta to objects cache",
|
|
||||||
zap.Stringer("bucket id", bktInfo.CID),
|
|
||||||
zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return conf, nil
|
return conf, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -185,7 +185,7 @@ func ParseCompletedPartHeader(hdr string) (*Part, error) {
|
||||||
|
|
||||||
// PutObject stores object into NeoFS, took payload from io.Reader.
|
// PutObject stores object into NeoFS, took payload from io.Reader.
|
||||||
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) {
|
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) {
|
||||||
own := n.Owner(ctx)
|
owner := n.Owner(ctx)
|
||||||
|
|
||||||
bktSettings, err := n.GetBucketSettings(ctx, p.BktInfo)
|
bktSettings, err := n.GetBucketSettings(ctx, p.BktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -230,7 +230,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
|
||||||
|
|
||||||
prm := PrmObjectCreate{
|
prm := PrmObjectCreate{
|
||||||
Container: p.BktInfo.CID,
|
Container: p.BktInfo.CID,
|
||||||
Creator: own,
|
Creator: owner,
|
||||||
PayloadSize: uint64(p.Size),
|
PayloadSize: uint64(p.Size),
|
||||||
Filepath: p.Object,
|
Filepath: p.Object,
|
||||||
Payload: r,
|
Payload: r,
|
||||||
|
@ -271,13 +271,13 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
n.listsCache.CleanCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
n.cache.CleanListCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
||||||
|
|
||||||
objInfo := &data.ObjectInfo{
|
objInfo := &data.ObjectInfo{
|
||||||
ID: id,
|
ID: id,
|
||||||
CID: p.BktInfo.CID,
|
CID: p.BktInfo.CID,
|
||||||
|
|
||||||
Owner: own,
|
Owner: owner,
|
||||||
Bucket: p.BktInfo.Name,
|
Bucket: p.BktInfo.Name,
|
||||||
Name: p.Object,
|
Name: p.Object,
|
||||||
Size: p.Size,
|
Size: p.Size,
|
||||||
|
@ -292,25 +292,15 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
|
||||||
NodeVersion: newVersion,
|
NodeVersion: newVersion,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.objCache.PutObject(extendedObjInfo); err != nil {
|
n.cache.PutObjectWithName(owner, extendedObjInfo)
|
||||||
n.log.Warn("couldn't add object to cache", zap.Error(err),
|
|
||||||
zap.String("object_name", p.Object), zap.String("bucket_name", p.BktInfo.Name),
|
|
||||||
zap.String("cid", objInfo.CID.EncodeToString()), zap.String("oid", objInfo.ID.EncodeToString()))
|
|
||||||
}
|
|
||||||
if err = n.namesCache.Put(objInfo.NiceName(), objInfo.Address()); err != nil {
|
|
||||||
n.log.Warn("couldn't put obj address to name cache",
|
|
||||||
zap.String("obj nice name", objInfo.NiceName()),
|
|
||||||
zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return objInfo, nil
|
return objInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ExtendedObjectInfo, error) {
|
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ExtendedObjectInfo, error) {
|
||||||
if addr := n.namesCache.Get(bkt.Name + "/" + objectName); addr != nil {
|
owner := n.Owner(ctx)
|
||||||
if extObjInfo := n.objCache.GetObject(*addr); extObjInfo != nil {
|
if extObjInfo := n.cache.GetLastObject(owner, bkt.Name, objectName); extObjInfo != nil {
|
||||||
return extObjInfo, nil
|
return extObjInfo, nil
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
node, err := n.treeService.GetLatestVersion(ctx, bkt, objectName)
|
node, err := n.treeService.GetLatestVersion(ctx, bkt, objectName)
|
||||||
|
@ -336,17 +326,7 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
|
||||||
NodeVersion: node,
|
NodeVersion: node,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.objCache.PutObject(extObjInfo); err != nil {
|
n.cache.PutObjectWithName(owner, extObjInfo)
|
||||||
n.log.Warn("couldn't put object info to cache",
|
|
||||||
zap.Stringer("object id", node.OID),
|
|
||||||
zap.Stringer("bucket id", bkt.CID),
|
|
||||||
zap.Error(err))
|
|
||||||
}
|
|
||||||
if err = n.namesCache.Put(objInfo.NiceName(), objInfo.Address()); err != nil {
|
|
||||||
n.log.Warn("couldn't put obj address to head cache",
|
|
||||||
zap.String("obj nice name", objInfo.NiceName()),
|
|
||||||
zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return extObjInfo, nil
|
return extObjInfo, nil
|
||||||
}
|
}
|
||||||
|
@ -379,7 +359,8 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if extObjInfo := n.objCache.GetObject(newAddress(bkt.CID, foundVersion.OID)); extObjInfo != nil {
|
owner := n.Owner(ctx)
|
||||||
|
if extObjInfo := n.cache.GetObject(owner, newAddress(bkt.CID, foundVersion.OID)); extObjInfo != nil {
|
||||||
return extObjInfo, nil
|
return extObjInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -397,14 +378,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
||||||
NodeVersion: foundVersion,
|
NodeVersion: foundVersion,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.objCache.PutObject(extObjInfo); err != nil {
|
n.cache.PutObject(owner, extObjInfo)
|
||||||
n.log.Warn("couldn't put obj to object cache",
|
|
||||||
zap.String("bucket name", objInfo.Bucket),
|
|
||||||
zap.Stringer("bucket cid", objInfo.CID),
|
|
||||||
zap.String("object name", objInfo.Name),
|
|
||||||
zap.Stringer("object id", objInfo.ID),
|
|
||||||
zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return extObjInfo, nil
|
return extObjInfo, nil
|
||||||
}
|
}
|
||||||
|
@ -418,7 +392,7 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
|
||||||
|
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
|
|
||||||
n.objCache.Delete(newAddress(bktInfo.CID, idObj))
|
n.cache.DeleteObject(newAddress(bktInfo.CID, idObj))
|
||||||
|
|
||||||
return n.neoFS.DeleteObject(ctx, prm)
|
return n.neoFS.DeleteObject(ctx, prm)
|
||||||
}
|
}
|
||||||
|
@ -503,17 +477,16 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams)
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
owner := n.Owner(ctx)
|
||||||
cacheKey := cache.CreateObjectsListCacheKey(p.Bucket.CID, p.Prefix, true)
|
cacheKey := cache.CreateObjectsListCacheKey(p.Bucket.CID, p.Prefix, true)
|
||||||
nodeVersions := n.listsCache.GetVersions(cacheKey)
|
nodeVersions := n.cache.GetList(owner, cacheKey)
|
||||||
|
|
||||||
if nodeVersions == nil {
|
if nodeVersions == nil {
|
||||||
nodeVersions, err = n.treeService.GetLatestVersionsByPrefix(ctx, p.Bucket, p.Prefix)
|
nodeVersions, err = n.treeService.GetLatestVersionsByPrefix(ctx, p.Bucket, p.Prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
if err = n.listsCache.PutVersions(cacheKey, nodeVersions); err != nil {
|
n.cache.PutList(owner, cacheKey, nodeVersions)
|
||||||
n.log.Error("couldn't cache list of objects", zap.Error(err))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(nodeVersions) == 0 {
|
if len(nodeVersions) == 0 {
|
||||||
|
@ -643,17 +616,17 @@ func getPartialObjectInfo(bktInfo *data.BucketInfo, node *data.NodeVersion) *dat
|
||||||
func (n *layer) bucketNodeVersions(ctx context.Context, bkt *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) {
|
func (n *layer) bucketNodeVersions(ctx context.Context, bkt *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
|
owner := n.Owner(ctx)
|
||||||
cacheKey := cache.CreateObjectsListCacheKey(bkt.CID, prefix, false)
|
cacheKey := cache.CreateObjectsListCacheKey(bkt.CID, prefix, false)
|
||||||
nodeVersions := n.listsCache.GetVersions(cacheKey)
|
nodeVersions := n.cache.GetList(owner, cacheKey)
|
||||||
|
|
||||||
if nodeVersions == nil {
|
if nodeVersions == nil {
|
||||||
nodeVersions, err = n.treeService.GetAllVersionsByPrefix(ctx, bkt, prefix)
|
nodeVersions, err = n.treeService.GetAllVersionsByPrefix(ctx, bkt, prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("get all versions from tree service: %w", err)
|
return nil, fmt.Errorf("get all versions from tree service: %w", err)
|
||||||
}
|
}
|
||||||
if err = n.listsCache.PutVersions(cacheKey, nodeVersions); err != nil {
|
|
||||||
n.log.Error("couldn't cache list of objects", zap.Error(err))
|
n.cache.PutList(owner, cacheKey, nodeVersions)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nodeVersions, nil
|
return nodeVersions, nil
|
||||||
|
@ -763,9 +736,9 @@ func (n *layer) objectInfoFromObjectsCacheOrNeoFS(ctx context.Context, bktInfo *
|
||||||
return oiDir
|
return oiDir
|
||||||
}
|
}
|
||||||
|
|
||||||
extObjInfo := n.objCache.GetObject(newAddress(bktInfo.CID, node.OID))
|
owner := n.Owner(ctx)
|
||||||
if extObjInfo != nil {
|
if extInfo := n.cache.GetObject(owner, newAddress(bktInfo.CID, node.OID)); extInfo != nil {
|
||||||
return extObjInfo.ObjectInfo
|
return extInfo.ObjectInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
meta, err := n.objectHead(ctx, bktInfo, node.OID)
|
meta, err := n.objectHead(ctx, bktInfo, node.OID)
|
||||||
|
@ -775,9 +748,7 @@ func (n *layer) objectInfoFromObjectsCacheOrNeoFS(ctx context.Context, bktInfo *
|
||||||
}
|
}
|
||||||
|
|
||||||
oi = objectInfoFromMeta(bktInfo, meta)
|
oi = objectInfoFromMeta(bktInfo, meta)
|
||||||
if err = n.objCache.PutObject(&data.ExtendedObjectInfo{ObjectInfo: oi, NodeVersion: node}); err != nil {
|
n.cache.PutObject(owner, &data.ExtendedObjectInfo{ObjectInfo: oi, NodeVersion: node})
|
||||||
n.log.Warn("couldn't cache an object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return oi
|
return oi
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,6 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -34,7 +33,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
|
||||||
// if not, then receive node version from tree service
|
// if not, then receive node version from tree service
|
||||||
if versionNode == nil {
|
if versionNode == nil {
|
||||||
// check cache if node version is stored inside extendedObjectVersion
|
// check cache if node version is stored inside extendedObjectVersion
|
||||||
versionNode = n.getNodeVersionFromCache(p.ObjVersion)
|
versionNode = n.getNodeVersionFromCache(n.Owner(ctx), p.ObjVersion)
|
||||||
if versionNode == nil {
|
if versionNode == nil {
|
||||||
// else get node version from tree service
|
// else get node version from tree service
|
||||||
versionNode, err = n.getNodeVersion(ctx, p.ObjVersion)
|
versionNode, err = n.getNodeVersion(ctx, p.ObjVersion)
|
||||||
|
@ -101,9 +100,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
|
||||||
return fmt.Errorf("couldn't put lock into tree: %w", err)
|
return fmt.Errorf("couldn't put lock into tree: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutLockInfo(lockObjectKey(p.ObjVersion), lockInfo); err != nil {
|
n.cache.PutLockInfo(lockObjectKey(p.ObjVersion), lockInfo)
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -127,7 +124,7 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) {
|
func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) {
|
||||||
if lockInfo := n.systemCache.GetLockInfo(lockObjectKey(objVersion)); lockInfo != nil {
|
if lockInfo := n.cache.GetLockInfo(lockObjectKey(objVersion)); lockInfo != nil {
|
||||||
return lockInfo, nil
|
return lockInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -144,17 +141,16 @@ func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*da
|
||||||
lockInfo = &data.LockInfo{}
|
lockInfo = &data.LockInfo{}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutLockInfo(lockObjectKey(objVersion), lockInfo); err != nil {
|
n.cache.PutLockInfo(lockObjectKey(objVersion), lockInfo)
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return lockInfo, nil
|
return lockInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo, sysName string) (*data.CORSConfiguration, error) {
|
func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSConfiguration, error) {
|
||||||
if cors := n.systemCache.GetCORS(systemObjectKey(bkt, sysName)); cors != nil {
|
if cors := n.cache.GetCORS(bkt); cors != nil {
|
||||||
return cors, nil
|
return cors, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
objID, err := n.treeService.GetBucketCORS(ctx, bkt)
|
objID, err := n.treeService.GetBucketCORS(ctx, bkt)
|
||||||
objIDNotFound := errorsStd.Is(err, ErrNodeNotFound)
|
objIDNotFound := errorsStd.Is(err, ErrNodeNotFound)
|
||||||
if err != nil && !objIDNotFound {
|
if err != nil && !objIDNotFound {
|
||||||
|
@ -176,30 +172,18 @@ func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo, sysName strin
|
||||||
return nil, fmt.Errorf("unmarshal cors: %w", err)
|
return nil, fmt.Errorf("unmarshal cors: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutCORS(systemObjectKey(bkt, sysName), cors); err != nil {
|
n.cache.PutCORS(bkt, cors)
|
||||||
objID, _ := obj.ID()
|
|
||||||
n.log.Warn("couldn't put system meta to objects cache",
|
|
||||||
zap.Stringer("object id", &objID),
|
|
||||||
zap.String("bucket id", bkt.CID.EncodeToString()),
|
|
||||||
zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return cors, nil
|
return cors, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// systemObjectKey is a key to use in SystemCache.
|
|
||||||
func systemObjectKey(bktInfo *data.BucketInfo, obj string) string {
|
|
||||||
return bktInfo.Name + obj
|
|
||||||
}
|
|
||||||
|
|
||||||
func lockObjectKey(objVersion *ObjectVersion) string {
|
func lockObjectKey(objVersion *ObjectVersion) string {
|
||||||
// todo reconsider forming name since versionID can be "null" or ""
|
// todo reconsider forming name since versionID can be "null" or ""
|
||||||
return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID
|
return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
func (n *layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
||||||
systemKey := systemObjectKey(bktInfo, bktInfo.SettingsObjectName())
|
if settings := n.cache.GetSettings(bktInfo); settings != nil {
|
||||||
if settings := n.systemCache.GetSettings(systemKey); settings != nil {
|
|
||||||
return settings, nil
|
return settings, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -211,11 +195,7 @@ func (n *layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
|
||||||
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}
|
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutSettings(systemKey, settings); err != nil {
|
n.cache.PutSettings(bktInfo, settings)
|
||||||
n.log.Warn("couldn't put system meta to objects cache",
|
|
||||||
zap.String("bucket id", bktInfo.CID.EncodeToString()),
|
|
||||||
zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return settings, nil
|
return settings, nil
|
||||||
}
|
}
|
||||||
|
@ -225,10 +205,7 @@ func (n *layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) err
|
||||||
return fmt.Errorf("failed to get settings node: %w", err)
|
return fmt.Errorf("failed to get settings node: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
systemKey := systemObjectKey(p.BktInfo, p.BktInfo.SettingsObjectName())
|
n.cache.PutSettings(p.BktInfo, p.Settings)
|
||||||
if err := n.systemCache.PutSettings(systemKey, p.Settings); err != nil {
|
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||||
"go.uber.org/zap"
|
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *layer) GetObjectTagging(ctx context.Context, p *ObjectVersion) (string, map[string]string, error) {
|
func (n *layer) GetObjectTagging(ctx context.Context, p *ObjectVersion) (string, map[string]string, error) {
|
||||||
|
@ -18,7 +18,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *ObjectVersion) (string,
|
||||||
)
|
)
|
||||||
|
|
||||||
if len(p.VersionID) != 0 && p.VersionID != data.UnversionedObjectVersionID {
|
if len(p.VersionID) != 0 && p.VersionID != data.UnversionedObjectVersionID {
|
||||||
tags = n.systemCache.GetTagging(objectTaggingCacheKey(p))
|
tags = n.cache.GetTagging(objectTaggingCacheKey(p))
|
||||||
if tags != nil {
|
if tags != nil {
|
||||||
return p.VersionID, tags, nil
|
return p.VersionID, tags, nil
|
||||||
}
|
}
|
||||||
|
@ -30,7 +30,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *ObjectVersion) (string,
|
||||||
}
|
}
|
||||||
p.VersionID = version.OID.EncodeToString()
|
p.VersionID = version.OID.EncodeToString()
|
||||||
|
|
||||||
tags = n.systemCache.GetTagging(objectTaggingCacheKey(p))
|
tags = n.cache.GetTagging(objectTaggingCacheKey(p))
|
||||||
if tags != nil {
|
if tags != nil {
|
||||||
return p.VersionID, tags, nil
|
return p.VersionID, tags, nil
|
||||||
}
|
}
|
||||||
|
@ -43,9 +43,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *ObjectVersion) (string,
|
||||||
return "", nil, err
|
return "", nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutTagging(objectTaggingCacheKey(p), tags); err != nil {
|
n.cache.PutTagging(objectTaggingCacheKey(p), tags)
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return p.VersionID, tags, nil
|
return p.VersionID, tags, nil
|
||||||
}
|
}
|
||||||
|
@ -65,9 +63,7 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *ObjectVersion, tagSet m
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = n.systemCache.PutTagging(objectTaggingCacheKey(p), tagSet); err != nil {
|
n.cache.PutTagging(objectTaggingCacheKey(p), tagSet)
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return version, nil
|
return version, nil
|
||||||
}
|
}
|
||||||
|
@ -88,7 +84,7 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*dat
|
||||||
|
|
||||||
p.VersionID = version.OID.EncodeToString()
|
p.VersionID = version.OID.EncodeToString()
|
||||||
|
|
||||||
n.systemCache.Delete(objectTaggingCacheKey(p))
|
n.cache.DeleteTagging(objectTaggingCacheKey(p))
|
||||||
|
|
||||||
return version, nil
|
return version, nil
|
||||||
}
|
}
|
||||||
|
@ -99,7 +95,7 @@ func (n *layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo)
|
||||||
tags map[string]string
|
tags map[string]string
|
||||||
)
|
)
|
||||||
|
|
||||||
tags = n.systemCache.GetTagging(bucketTaggingCacheKey(bktInfo.CID))
|
tags = n.cache.GetTagging(bucketTaggingCacheKey(bktInfo.CID))
|
||||||
if tags != nil {
|
if tags != nil {
|
||||||
return tags, nil
|
return tags, nil
|
||||||
}
|
}
|
||||||
|
@ -108,9 +104,7 @@ func (n *layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := n.systemCache.PutTagging(bucketTaggingCacheKey(bktInfo.CID), tags); err != nil {
|
n.cache.PutTagging(bucketTaggingCacheKey(bktInfo.CID), tags)
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return tags, nil
|
return tags, nil
|
||||||
}
|
}
|
||||||
|
@ -119,15 +113,14 @@ func (n *layer) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
if err := n.treeService.PutBucketTagging(ctx, bktInfo, tagSet); err != nil {
|
if err := n.treeService.PutBucketTagging(ctx, bktInfo, tagSet); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := n.systemCache.PutTagging(bucketTaggingCacheKey(bktInfo.CID), tagSet); err != nil {
|
|
||||||
n.log.Error("couldn't cache system object", zap.Error(err))
|
n.cache.PutTagging(bucketTaggingCacheKey(bktInfo.CID), tagSet)
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error {
|
func (n *layer) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
n.systemCache.Delete(bucketTaggingCacheKey(bktInfo.CID))
|
n.cache.DeleteTagging(bucketTaggingCacheKey(bktInfo.CID))
|
||||||
|
|
||||||
return n.treeService.DeleteBucketTagging(ctx, bktInfo)
|
return n.treeService.DeleteBucketTagging(ctx, bktInfo)
|
||||||
}
|
}
|
||||||
|
@ -171,7 +164,7 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
|
||||||
return version, err
|
return version, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) getNodeVersionFromCache(o *ObjectVersion) *data.NodeVersion {
|
func (n *layer) getNodeVersionFromCache(owner user.ID, o *ObjectVersion) *data.NodeVersion {
|
||||||
if len(o.VersionID) == 0 || o.VersionID == data.UnversionedObjectVersionID {
|
if len(o.VersionID) == 0 || o.VersionID == data.UnversionedObjectVersionID {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -185,7 +178,7 @@ func (n *layer) getNodeVersionFromCache(o *ObjectVersion) *data.NodeVersion {
|
||||||
addr.SetContainer(o.BktInfo.CID)
|
addr.SetContainer(o.BktInfo.CID)
|
||||||
addr.SetObject(objID)
|
addr.SetObject(objID)
|
||||||
|
|
||||||
extObjectInfo := n.objCache.GetObject(addr)
|
extObjectInfo := n.cache.GetObject(owner, addr)
|
||||||
if extObjectInfo == nil {
|
if extObjectInfo == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -547,9 +547,12 @@ func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CachesConfig {
|
||||||
cacheCfg.Names.Lifetime = getLifetime(v, l, cfgNamesCacheLifetime, cacheCfg.Names.Lifetime)
|
cacheCfg.Names.Lifetime = getLifetime(v, l, cfgNamesCacheLifetime, cacheCfg.Names.Lifetime)
|
||||||
cacheCfg.Names.Size = getSize(v, l, cfgNamesCacheSize, cacheCfg.Names.Size)
|
cacheCfg.Names.Size = getSize(v, l, cfgNamesCacheSize, cacheCfg.Names.Size)
|
||||||
|
|
||||||
cacheCfg.System.Lifetime = getLifetime(v, l, cfgSystemLifetimeSize, cacheCfg.System.Lifetime)
|
cacheCfg.System.Lifetime = getLifetime(v, l, cfgSystemCacheLifetime, cacheCfg.System.Lifetime)
|
||||||
cacheCfg.System.Size = getSize(v, l, cfgSystemCacheSize, cacheCfg.System.Size)
|
cacheCfg.System.Size = getSize(v, l, cfgSystemCacheSize, cacheCfg.System.Size)
|
||||||
|
|
||||||
|
cacheCfg.AccessControl.Lifetime = getLifetime(v, l, cfgAccessControlCacheLifetime, cacheCfg.AccessControl.Lifetime)
|
||||||
|
cacheCfg.AccessControl.Size = getSize(v, l, cfgAccessControlCacheSize, cacheCfg.AccessControl.Size)
|
||||||
|
|
||||||
return cacheCfg
|
return cacheCfg
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -52,18 +52,20 @@ const ( // Settings.
|
||||||
cfgPoolErrorThreshold = "pool_error_threshold"
|
cfgPoolErrorThreshold = "pool_error_threshold"
|
||||||
|
|
||||||
// Caching.
|
// Caching.
|
||||||
cfgObjectsCacheLifetime = "cache.objects.lifetime"
|
cfgObjectsCacheLifetime = "cache.objects.lifetime"
|
||||||
cfgObjectsCacheSize = "cache.objects.size"
|
cfgObjectsCacheSize = "cache.objects.size"
|
||||||
cfgListObjectsCacheLifetime = "cache.list.lifetime"
|
cfgListObjectsCacheLifetime = "cache.list.lifetime"
|
||||||
cfgListObjectsCacheSize = "cache.list.size"
|
cfgListObjectsCacheSize = "cache.list.size"
|
||||||
cfgBucketsCacheLifetime = "cache.buckets.lifetime"
|
cfgBucketsCacheLifetime = "cache.buckets.lifetime"
|
||||||
cfgBucketsCacheSize = "cache.buckets.size"
|
cfgBucketsCacheSize = "cache.buckets.size"
|
||||||
cfgNamesCacheLifetime = "cache.names.lifetime"
|
cfgNamesCacheLifetime = "cache.names.lifetime"
|
||||||
cfgNamesCacheSize = "cache.names.size"
|
cfgNamesCacheSize = "cache.names.size"
|
||||||
cfgSystemLifetimeSize = "cache.system.lifetime"
|
cfgSystemCacheLifetime = "cache.system.lifetime"
|
||||||
cfgSystemCacheSize = "cache.system.size"
|
cfgSystemCacheSize = "cache.system.size"
|
||||||
cfgAccessBoxCacheLifetime = "cache.accessbox.lifetime"
|
cfgAccessBoxCacheLifetime = "cache.accessbox.lifetime"
|
||||||
cfgAccessBoxCacheSize = "cache.accessbox.size"
|
cfgAccessBoxCacheSize = "cache.accessbox.size"
|
||||||
|
cfgAccessControlCacheLifetime = "cache.accesscontrol.lifetime"
|
||||||
|
cfgAccessControlCacheSize = "cache.accesscontrol.size"
|
||||||
|
|
||||||
// NATS.
|
// NATS.
|
||||||
cfgEnableNATS = "nats.enabled"
|
cfgEnableNATS = "nats.enabled"
|
||||||
|
|
Loading…
Reference in a new issue