TrueCloudLab
/
frostfs-s3-gw
16
2
Fork 14
Code Issues 19 Pull Requests 6 Actions Packages Releases 7 Activity

[#547] Cache ObjectInfo instead of Object 

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
remotes/KirillovDenis/bugfix/681-fix_acl_parsing
Denis Kirillov 2022-06-24 12:01:39 +03:00 committed by Alex Vanin
parent 127b91a97f
commit 42a761c1f2
3 changed files with 37 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
api/cache/objects.go vendored
View File

@ -6,7 +6,6 @@ import (
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)
@ -39,23 +38,6 @@ func New(config *Config) *ObjectsCache {
return &ObjectsCache{cache: gc, logger: config.Logger}
}
// Get returns a cached object.
func (o *ObjectsCache) Get(address oid.Address) *object.Object {
entry, err := o.cache.Get(address.EncodeToString())
if err != nil {
return nil
}
result, ok := entry.(object.Object)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}
return &result
}
// GetObject returns a cached object info.
func (o *ObjectsCache) GetObject(address oid.Address) *data.ObjectInfo {
entry, err := o.cache.Get(address.EncodeToString())
@ -65,30 +47,14 @@ func (o *ObjectsCache) GetObject(address oid.Address) *data.ObjectInfo {
result, ok := entry.(*data.ObjectInfo)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}
return result
}
// Put puts an object to cache.
func (o *ObjectsCache) Put(obj object.Object) error {
cnrID, ok := obj.ContainerID()
if !ok {
return fmt.Errorf("empty container id")
}
objID, ok := obj.ID()
if !ok {
return fmt.Errorf("empty object id")
}
var addr oid.Address
addr.SetContainer(cnrID)
addr.SetObject(objID)
return o.cache.Set(addr.EncodeToString(), obj)
}
// PutObject puts an object info to cache.
func (o *ObjectsCache) PutObject(obj *data.ObjectInfo) error {
cnrID := obj.CID.EncodeToString()

7
api/layer/multipart_upload.go
View File

@ -228,10 +228,6 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
HashSum: partInfo.ETag,
}
if err = n.objCache.PutObject(objInfo); err != nil {
n.log.Error("couldn't cache system object", zap.Error(err))
}
return objInfo, nil
}
@ -335,6 +331,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
return nil, nil, errors.GetAPIError(errors.ErrInvalidPart)
}
var multipartObjetSize int64
parts := make([]*data.PartInfo, 0, len(p.Parts))
var completedPartsHeader strings.Builder
@ -348,6 +345,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
return nil, nil, errors.GetAPIError(errors.ErrEntityTooSmall)
}
parts = append(parts, partInfo)
multipartObjetSize += partInfo.Size
partInfoStr := partInfo.ToHeaderString()
if i != len(p.Parts)-1 {
@ -388,6 +386,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
Object: p.Info.Key,
Reader: r,
Header: initMetadata,
Size: multipartObjetSize,
})
if err != nil {
n.log.Error("could not put a completed object (multipart upload)",

68
api/layer/object.go
View File

@ -238,8 +238,8 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ObjectInfo, error) {
if addr := n.namesCache.Get(bkt.Name + "/" + objectName); addr != nil {
if headInfo := n.objCache.Get(*addr); headInfo != nil {
return objInfoFromMeta(bkt, headInfo), nil
if objInfo := n.objCache.GetObject(*addr); objInfo != nil {
return objInfo, nil
}
}
@ -259,14 +259,13 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
if err != nil {
return nil, err
}
if err = n.objCache.Put(*meta); err != nil {
n.log.Warn("couldn't put meta to objects cache",
objInfo := objInfoFromMeta(bkt, meta)
if err = n.objCache.PutObject(objInfo); err != nil {
n.log.Warn("couldn't put object info to cache",
zap.Stringer("object id", node.OID),
zap.Stringer("bucket id", bkt.CID),
zap.Error(err))
}
objInfo := objInfoFromMeta(bkt, meta)
if err = n.namesCache.Put(objInfo.NiceName(), objInfo.Address()); err != nil {
n.log.Warn("couldn't put obj address to head cache",
zap.String("obj nice name", objInfo.NiceName()),
@ -304,8 +303,8 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
}
}
if headInfo := n.objCache.Get(newAddress(bkt.CID, foundVersion.OID)); headInfo != nil {
return objInfoFromMeta(bkt, headInfo), nil
if objInfo := n.objCache.GetObject(newAddress(bkt.CID, foundVersion.OID)); objInfo != nil {
return objInfo, nil
}
meta, err := n.objectHead(ctx, bkt, foundVersion.OID)
@ -317,7 +316,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
}
objInfo := objInfoFromMeta(bkt, meta)
if err = n.objCache.Put(*meta); err != nil {
if err = n.objCache.PutObject(objInfo); err != nil {
n.log.Warn("couldn't put obj to object cache",
zap.String("bucket name", objInfo.Bucket),
zap.Stringer("bucket cid", objInfo.CID),
@ -436,6 +435,10 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams)
}
}
if len(nodeVersions) == 0 {
return nil, nil, nil
}
sort.Slice(nodeVersions, func(i, j int) bool {
return nodeVersions[i].FilePath < nodeVersions[j].FilePath
})
@ -511,12 +514,10 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p allObjectParams,
wg.Add(1)
err = pool.Submit(func() {
defer wg.Done()
if obj := n.objectFromObjectsCacheOrNeoFS(ctx, p.Bucket, node.OID); obj != nil {
if oi := objectInfoFromMeta(p.Bucket, obj, p.Prefix, p.Delimiter); oi != nil {
select {
case <-ctx.Done():
case objCh <- oi:
}
if oi := n.objectInfoFromObjectsCacheOrNeoFS(ctx, p.Bucket, node.OID, p.Prefix, p.Delimiter); oi != nil {
select {
case <-ctx.Done():
case objCh <- oi:
}
}
})
@ -562,12 +563,7 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
oi.Created = nodeVersion.DeleteMarker.Created
oi.IsDeleteMarker = true
} else {
obj := n.objectFromObjectsCacheOrNeoFS(ctx, bkt, nodeVersion.OID)
if obj == nil {
continue
}
oi = objectInfoFromMeta(bkt, obj, prefix, delimiter)
if oi == nil {
if oi = n.objectInfoFromObjectsCacheOrNeoFS(ctx, bkt, nodeVersion.OID, prefix, delimiter); oi == nil {
continue
}
}
@ -657,23 +653,23 @@ func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *data.BucketInf
return settings.VersioningEnabled
}
func (n *layer) objectFromObjectsCacheOrNeoFS(ctx context.Context, bktInfo *data.BucketInfo, obj oid.ID) *object.Object {
var (
err error
meta = n.objCache.Get(newAddress(bktInfo.CID, obj))
)
if meta == nil {
meta, err = n.objectHead(ctx, bktInfo, obj)
if err != nil {
n.log.Warn("could not fetch object meta", zap.Error(err))
return nil
}
if err = n.objCache.Put(*meta); err != nil {
n.log.Error("couldn't cache an object", zap.Error(err))
}
func (n *layer) objectInfoFromObjectsCacheOrNeoFS(ctx context.Context, bktInfo *data.BucketInfo, obj oid.ID, prefix, delimiter string) *data.ObjectInfo {
if objInfo := n.objCache.GetObject(newAddress(bktInfo.CID, obj)); objInfo != nil {
return objInfo
}
return meta
meta, err := n.objectHead(ctx, bktInfo, obj)
if err != nil {
n.log.Warn("could not fetch object meta", zap.Error(err))
return nil
}
objInfo := objectInfoFromMeta(bktInfo, meta, prefix, delimiter)
if err = n.objCache.PutObject(objInfo); err != nil {
n.log.Warn("couldn't cache an object", zap.Error(err))
}
return objInfo
}
func (n *layer) transformNeofsError(ctx context.Context, err error) error {