2020-07-03 13:52:04 +00:00
|
|
|
package layer
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2022-05-19 14:56:42 +00:00
|
|
|
"crypto/sha256"
|
2022-05-12 16:58:11 +00:00
|
|
|
"encoding/hex"
|
2022-02-08 16:54:04 +00:00
|
|
|
"errors"
|
2022-03-01 15:07:15 +00:00
|
|
|
"fmt"
|
2020-07-03 13:52:04 +00:00
|
|
|
"io"
|
2022-06-29 10:51:05 +00:00
|
|
|
"mime"
|
|
|
|
"path/filepath"
|
2021-07-18 13:40:19 +00:00
|
|
|
"sort"
|
2021-08-10 10:03:09 +00:00
|
|
|
"strings"
|
2020-07-03 13:52:04 +00:00
|
|
|
"time"
|
|
|
|
|
2021-08-18 13:48:58 +00:00
|
|
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
2021-08-27 22:20:40 +00:00
|
|
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
2021-09-10 06:56:56 +00:00
|
|
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
2021-08-09 08:53:58 +00:00
|
|
|
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
2022-04-21 08:49:56 +00:00
|
|
|
"github.com/nspcc-dev/neofs-sdk-go/client"
|
2021-11-15 12:56:16 +00:00
|
|
|
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
|
|
|
"github.com/nspcc-dev/neofs-sdk-go/object"
|
2022-02-08 16:54:04 +00:00
|
|
|
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
2021-07-18 13:40:19 +00:00
|
|
|
"go.uber.org/zap"
|
2020-07-03 13:52:04 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type (
|
2020-10-19 01:04:37 +00:00
|
|
|
getParams struct {
|
2022-02-08 16:54:04 +00:00
|
|
|
// payload range
|
|
|
|
off, ln uint64
|
|
|
|
|
2022-05-31 12:38:06 +00:00
|
|
|
oid oid.ID
|
2022-06-02 16:56:04 +00:00
|
|
|
bktInfo *data.BucketInfo
|
2020-07-03 13:52:04 +00:00
|
|
|
}
|
2021-07-18 13:40:19 +00:00
|
|
|
|
|
|
|
// ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2.
|
|
|
|
ListObjectsParamsCommon struct {
|
2022-03-18 13:04:09 +00:00
|
|
|
BktInfo *data.BucketInfo
|
2021-07-18 13:40:19 +00:00
|
|
|
Delimiter string
|
|
|
|
Encode string
|
|
|
|
MaxKeys int
|
|
|
|
Prefix string
|
|
|
|
}
|
|
|
|
|
|
|
|
// ListObjectsParamsV1 contains params for ListObjectsV1.
|
|
|
|
ListObjectsParamsV1 struct {
|
|
|
|
ListObjectsParamsCommon
|
|
|
|
Marker string
|
|
|
|
}
|
|
|
|
|
|
|
|
// ListObjectsParamsV2 contains params for ListObjectsV2.
|
|
|
|
ListObjectsParamsV2 struct {
|
|
|
|
ListObjectsParamsCommon
|
|
|
|
ContinuationToken string
|
|
|
|
StartAfter string
|
2021-07-20 12:40:38 +00:00
|
|
|
FetchOwner bool
|
2021-07-18 13:40:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
allObjectParams struct {
|
2022-06-10 11:57:41 +00:00
|
|
|
Bucket *data.BucketInfo
|
|
|
|
Delimiter string
|
|
|
|
Prefix string
|
|
|
|
MaxKeys int
|
|
|
|
Marker string
|
|
|
|
ContinuationToken string
|
2021-07-18 13:40:19 +00:00
|
|
|
}
|
2020-07-03 13:52:04 +00:00
|
|
|
)
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
const (
|
|
|
|
continuationToken = "<continuation-token>"
|
|
|
|
)
|
|
|
|
|
2022-05-25 17:25:43 +00:00
|
|
|
func newAddress(cnr cid.ID, obj oid.ID) oid.Address {
|
|
|
|
var addr oid.Address
|
|
|
|
addr.SetContainer(cnr)
|
|
|
|
addr.SetObject(obj)
|
2022-02-08 16:54:04 +00:00
|
|
|
return addr
|
2021-08-18 13:48:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// objectHead returns all object's headers.
|
2022-06-01 17:35:20 +00:00
|
|
|
func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) (*object.Object, error) {
|
2022-06-06 11:09:09 +00:00
|
|
|
prm := PrmObjectRead{
|
2022-06-01 17:35:20 +00:00
|
|
|
Container: bktInfo.CID,
|
2022-04-25 09:57:58 +00:00
|
|
|
Object: idObj,
|
2022-03-01 19:02:24 +00:00
|
|
|
WithHeader: true,
|
|
|
|
}
|
|
|
|
|
2022-06-01 17:35:20 +00:00
|
|
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
2022-02-08 16:54:04 +00:00
|
|
|
|
2022-03-01 19:02:24 +00:00
|
|
|
res, err := n.neoFS.ReadObject(ctx, prm)
|
|
|
|
if err != nil {
|
|
|
|
return nil, n.transformNeofsError(ctx, err)
|
|
|
|
}
|
2022-02-08 16:54:04 +00:00
|
|
|
|
2022-03-01 19:02:24 +00:00
|
|
|
return res.Head, nil
|
2020-08-05 13:47:09 +00:00
|
|
|
}
|
|
|
|
|
2022-03-02 16:09:02 +00:00
|
|
|
// initializes payload reader of the NeoFS object.
|
2022-02-08 16:54:04 +00:00
|
|
|
// Zero range corresponds to full payload (panics if only offset is set).
|
2022-03-02 16:09:02 +00:00
|
|
|
func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) {
|
2022-06-06 11:09:09 +00:00
|
|
|
prm := PrmObjectRead{
|
2022-05-31 12:38:06 +00:00
|
|
|
Container: p.bktInfo.CID,
|
|
|
|
Object: p.oid,
|
2022-03-01 19:02:24 +00:00
|
|
|
WithPayload: true,
|
|
|
|
PayloadRange: [2]uint64{p.off, p.ln},
|
2022-02-08 16:54:04 +00:00
|
|
|
}
|
|
|
|
|
2022-06-02 16:56:04 +00:00
|
|
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, p.bktInfo.Owner)
|
2022-02-08 16:54:04 +00:00
|
|
|
|
2022-03-01 19:02:24 +00:00
|
|
|
res, err := n.neoFS.ReadObject(ctx, prm)
|
2022-03-02 16:09:02 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, n.transformNeofsError(ctx, err)
|
2022-02-08 16:54:04 +00:00
|
|
|
}
|
|
|
|
|
2022-03-02 16:09:02 +00:00
|
|
|
return res.Payload, nil
|
2021-10-04 14:32:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// objectGet returns an object with payload in the object.
|
2022-06-01 17:35:20 +00:00
|
|
|
func (n *layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (*object.Object, error) {
|
2022-06-06 11:09:09 +00:00
|
|
|
prm := PrmObjectRead{
|
2022-06-01 17:35:20 +00:00
|
|
|
Container: bktInfo.CID,
|
|
|
|
Object: objID,
|
2022-03-01 19:02:24 +00:00
|
|
|
WithHeader: true,
|
|
|
|
WithPayload: true,
|
2022-02-08 16:54:04 +00:00
|
|
|
}
|
|
|
|
|
2022-06-01 17:35:20 +00:00
|
|
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
2022-02-08 16:54:04 +00:00
|
|
|
|
2022-03-01 19:02:24 +00:00
|
|
|
res, err := n.neoFS.ReadObject(ctx, prm)
|
2022-02-08 16:54:04 +00:00
|
|
|
if err != nil {
|
2022-03-01 19:02:24 +00:00
|
|
|
return nil, n.transformNeofsError(ctx, err)
|
2022-02-08 16:54:04 +00:00
|
|
|
}
|
|
|
|
|
2022-03-01 19:02:24 +00:00
|
|
|
return res.Head, nil
|
2021-06-24 11:10:00 +00:00
|
|
|
}
|
|
|
|
|
2022-06-29 10:51:05 +00:00
|
|
|
// MimeByFileName detect mime type by filename extension.
|
|
|
|
func MimeByFileName(name string) string {
|
|
|
|
ext := filepath.Ext(name)
|
|
|
|
if len(ext) == 0 {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
return mime.TypeByExtension(ext)
|
|
|
|
}
|
|
|
|
|
2022-03-18 13:04:09 +00:00
|
|
|
// PutObject stores object into NeoFS, took payload from io.Reader.
|
|
|
|
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) {
|
2021-08-13 14:13:14 +00:00
|
|
|
own := n.Owner(ctx)
|
2021-07-12 12:04:27 +00:00
|
|
|
|
2022-03-18 13:04:09 +00:00
|
|
|
versioningEnabled := n.isVersioningEnabled(ctx, p.BktInfo)
|
2022-06-10 11:57:41 +00:00
|
|
|
newVersion := &data.NodeVersion{
|
|
|
|
BaseNodeVersion: data.BaseNodeVersion{FilePath: p.Object},
|
|
|
|
IsUnversioned: !versioningEnabled,
|
|
|
|
}
|
2020-07-03 13:52:04 +00:00
|
|
|
|
2021-08-18 13:48:58 +00:00
|
|
|
r := p.Reader
|
2021-11-13 20:35:50 +00:00
|
|
|
if r != nil {
|
|
|
|
if len(p.Header[api.ContentType]) == 0 {
|
2022-06-29 10:51:05 +00:00
|
|
|
if contentType := MimeByFileName(p.Object); len(contentType) == 0 {
|
|
|
|
d := newDetector(r)
|
2022-05-17 14:56:05 +00:00
|
|
|
if contentType, err := d.Detect(); err == nil {
|
2022-06-29 10:51:05 +00:00
|
|
|
p.Header[api.ContentType] = contentType
|
|
|
|
}
|
|
|
|
r = d.MultiReader()
|
|
|
|
} else {
|
2021-11-13 20:35:50 +00:00
|
|
|
p.Header[api.ContentType] = contentType
|
|
|
|
}
|
2021-08-18 13:48:58 +00:00
|
|
|
}
|
|
|
|
}
|
2021-08-10 08:19:09 +00:00
|
|
|
|
2022-06-06 11:09:09 +00:00
|
|
|
prm := PrmObjectCreate{
|
2022-05-25 17:25:43 +00:00
|
|
|
Container: p.BktInfo.CID,
|
2022-04-25 09:57:58 +00:00
|
|
|
Creator: own,
|
2022-03-01 19:02:24 +00:00
|
|
|
PayloadSize: uint64(p.Size),
|
|
|
|
Filename: p.Object,
|
|
|
|
Payload: r,
|
|
|
|
}
|
|
|
|
|
|
|
|
prm.Attributes = make([][2]string, 0, len(p.Header))
|
|
|
|
|
|
|
|
for k, v := range p.Header {
|
|
|
|
prm.Attributes = append(prm.Attributes, [2]string{k, v})
|
|
|
|
}
|
|
|
|
|
2022-06-01 17:35:20 +00:00
|
|
|
id, hash, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
2022-03-15 09:37:56 +00:00
|
|
|
if err != nil {
|
2022-04-12 06:52:44 +00:00
|
|
|
return nil, err
|
2022-03-15 09:37:56 +00:00
|
|
|
}
|
|
|
|
|
2022-05-18 06:51:12 +00:00
|
|
|
newVersion.OID = *id
|
2022-06-10 11:57:41 +00:00
|
|
|
if err = n.treeService.AddVersion(ctx, &p.BktInfo.CID, newVersion); err != nil {
|
2022-05-17 14:56:05 +00:00
|
|
|
return nil, fmt.Errorf("couldn't add new verion to tree service: %w", err)
|
|
|
|
}
|
|
|
|
|
2022-05-26 13:11:14 +00:00
|
|
|
if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) {
|
|
|
|
objVersion := &ObjectVersion{
|
|
|
|
BktInfo: p.BktInfo,
|
|
|
|
ObjectName: p.Object,
|
|
|
|
VersionID: id.EncodeToString(),
|
2022-03-01 15:07:15 +00:00
|
|
|
}
|
2022-05-26 13:11:14 +00:00
|
|
|
|
|
|
|
if err = n.PutLockInfo(ctx, objVersion, p.Lock); err != nil {
|
|
|
|
return nil, err
|
2022-03-01 15:07:15 +00:00
|
|
|
}
|
2022-02-28 10:22:07 +00:00
|
|
|
}
|
|
|
|
|
2022-03-18 13:04:09 +00:00
|
|
|
n.listsCache.CleanCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
2021-09-01 16:10:52 +00:00
|
|
|
|
2022-05-24 14:55:56 +00:00
|
|
|
objInfo := &data.ObjectInfo{
|
2022-05-25 17:25:43 +00:00
|
|
|
ID: *id,
|
2022-03-18 13:04:09 +00:00
|
|
|
CID: p.BktInfo.CID,
|
2021-08-13 14:13:14 +00:00
|
|
|
|
2022-06-03 14:33:47 +00:00
|
|
|
Owner: own,
|
|
|
|
Bucket: p.BktInfo.Name,
|
|
|
|
Name: p.Object,
|
|
|
|
Size: p.Size,
|
|
|
|
Created: time.Now(),
|
|
|
|
Headers: p.Header,
|
|
|
|
ContentType: p.Header[api.ContentType],
|
|
|
|
HashSum: hex.EncodeToString(hash),
|
2022-05-24 14:55:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if err = n.objCache.PutObject(objInfo); err != nil {
|
|
|
|
n.log.Warn("couldn't add object to cache", zap.Error(err),
|
|
|
|
zap.String("object_name", p.Object), zap.String("bucket_name", p.BktInfo.Name),
|
|
|
|
zap.String("cid", objInfo.CID.EncodeToString()), zap.String("oid", objInfo.ID.EncodeToString()))
|
|
|
|
}
|
|
|
|
if err = n.namesCache.Put(objInfo.NiceName(), objInfo.Address()); err != nil {
|
|
|
|
n.log.Warn("couldn't put obj address to name cache",
|
|
|
|
zap.String("obj nice name", objInfo.NiceName()),
|
|
|
|
zap.Error(err))
|
|
|
|
}
|
|
|
|
|
|
|
|
return objInfo, nil
|
2021-08-13 14:13:14 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 06:56:56 +00:00
|
|
|
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ObjectInfo, error) {
|
2022-02-08 16:54:04 +00:00
|
|
|
if addr := n.namesCache.Get(bkt.Name + "/" + objectName); addr != nil {
|
2022-05-25 17:25:43 +00:00
|
|
|
if headInfo := n.objCache.Get(*addr); headInfo != nil {
|
2021-08-18 13:48:58 +00:00
|
|
|
return objInfoFromMeta(bkt, headInfo), nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-17 14:56:05 +00:00
|
|
|
node, err := n.treeService.GetLatestVersion(ctx, &bkt.CID, objectName)
|
2021-08-13 14:13:14 +00:00
|
|
|
if err != nil {
|
2022-05-18 07:48:30 +00:00
|
|
|
if errors.Is(err, ErrNodeNotFound) {
|
|
|
|
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
|
|
|
|
}
|
2021-08-13 14:13:14 +00:00
|
|
|
return nil, err
|
2021-08-10 08:19:09 +00:00
|
|
|
}
|
|
|
|
|
2022-05-20 15:02:00 +00:00
|
|
|
if node.DeleteMarker != nil {
|
2021-08-13 14:13:14 +00:00
|
|
|
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
|
|
|
|
}
|
2021-08-18 13:48:58 +00:00
|
|
|
|
2022-05-18 06:51:12 +00:00
|
|
|
meta, err := n.objectHead(ctx, bkt, node.OID)
|
2022-05-17 14:56:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if err = n.objCache.Put(*meta); err != nil {
|
|
|
|
n.log.Warn("couldn't put meta to objects cache",
|
|
|
|
zap.Stringer("object id", node.OID),
|
|
|
|
zap.Stringer("bucket id", bkt.CID),
|
|
|
|
zap.Error(err))
|
|
|
|
}
|
|
|
|
|
|
|
|
objInfo := objInfoFromMeta(bkt, meta)
|
|
|
|
if err = n.namesCache.Put(objInfo.NiceName(), objInfo.Address()); err != nil {
|
2021-08-18 13:48:58 +00:00
|
|
|
n.log.Warn("couldn't put obj address to head cache",
|
2022-05-17 14:56:05 +00:00
|
|
|
zap.String("obj nice name", objInfo.NiceName()),
|
2021-08-18 13:48:58 +00:00
|
|
|
zap.Error(err))
|
|
|
|
}
|
|
|
|
|
2022-05-17 14:56:05 +00:00
|
|
|
return objInfo, nil
|
2021-08-10 08:19:09 +00:00
|
|
|
}
|
|
|
|
|
2022-01-18 09:40:41 +00:00
|
|
|
func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
2022-05-31 08:12:53 +00:00
|
|
|
var err error
|
|
|
|
var foundVersion *data.NodeVersion
|
2022-05-31 15:03:58 +00:00
|
|
|
if p.VersionID == UnversionedObjectVersionID {
|
2022-05-31 08:12:53 +00:00
|
|
|
foundVersion, err = n.treeService.GetUnversioned(ctx, &bkt.CID, p.Object)
|
2022-01-18 09:40:41 +00:00
|
|
|
if err != nil {
|
2022-05-31 08:12:53 +00:00
|
|
|
if errors.Is(err, ErrNodeNotFound) {
|
|
|
|
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion)
|
|
|
|
}
|
2022-01-18 09:40:41 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
2022-05-31 08:12:53 +00:00
|
|
|
} else {
|
|
|
|
versions, err := n.treeService.GetVersions(ctx, &bkt.CID, p.Object)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("couldn't get versions: %w", err)
|
2022-01-18 09:40:41 +00:00
|
|
|
}
|
|
|
|
|
2022-05-31 08:12:53 +00:00
|
|
|
for _, version := range versions {
|
|
|
|
if version.OID.EncodeToString() == p.VersionID {
|
|
|
|
foundVersion = version
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if foundVersion == nil {
|
|
|
|
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion)
|
2022-05-18 07:48:30 +00:00
|
|
|
}
|
2021-08-10 10:03:09 +00:00
|
|
|
}
|
|
|
|
|
2022-05-18 07:48:30 +00:00
|
|
|
if headInfo := n.objCache.Get(newAddress(bkt.CID, foundVersion.OID)); headInfo != nil {
|
2021-08-18 13:48:58 +00:00
|
|
|
return objInfoFromMeta(bkt, headInfo), nil
|
|
|
|
}
|
|
|
|
|
2022-05-18 07:48:30 +00:00
|
|
|
meta, err := n.objectHead(ctx, bkt, foundVersion.OID)
|
2021-08-10 12:08:15 +00:00
|
|
|
if err != nil {
|
2022-04-21 08:49:56 +00:00
|
|
|
if client.IsErrObjectNotFound(err) {
|
2021-08-11 10:02:13 +00:00
|
|
|
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion)
|
2021-08-10 10:03:09 +00:00
|
|
|
}
|
2021-08-10 12:08:15 +00:00
|
|
|
return nil, err
|
2021-08-10 10:03:09 +00:00
|
|
|
}
|
|
|
|
|
2021-10-04 14:30:38 +00:00
|
|
|
objInfo := objInfoFromMeta(bkt, meta)
|
2021-08-18 13:48:58 +00:00
|
|
|
if err = n.objCache.Put(*meta); err != nil {
|
|
|
|
n.log.Warn("couldn't put obj to object cache",
|
|
|
|
zap.String("bucket name", objInfo.Bucket),
|
2021-08-27 21:33:50 +00:00
|
|
|
zap.Stringer("bucket cid", objInfo.CID),
|
2021-08-18 13:48:58 +00:00
|
|
|
zap.String("object name", objInfo.Name),
|
2021-08-27 21:33:50 +00:00
|
|
|
zap.Stringer("object id", objInfo.ID),
|
2021-08-18 13:48:58 +00:00
|
|
|
zap.Error(err))
|
|
|
|
}
|
|
|
|
|
|
|
|
return objInfo, nil
|
2020-07-03 13:52:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// objectDelete puts tombstone object into neofs.
|
2022-06-01 17:35:20 +00:00
|
|
|
func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
|
2022-06-06 11:09:09 +00:00
|
|
|
prm := PrmObjectDelete{
|
2022-06-01 17:35:20 +00:00
|
|
|
Container: bktInfo.CID,
|
2022-05-25 17:25:43 +00:00
|
|
|
Object: idObj,
|
2022-03-01 19:02:24 +00:00
|
|
|
}
|
|
|
|
|
2022-06-01 17:35:20 +00:00
|
|
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
2022-03-01 19:02:24 +00:00
|
|
|
|
2022-06-01 17:35:20 +00:00
|
|
|
n.objCache.Delete(newAddress(bktInfo.CID, idObj))
|
2022-03-01 19:02:24 +00:00
|
|
|
|
|
|
|
return n.transformNeofsError(ctx, n.neoFS.DeleteObject(ctx, prm))
|
2020-07-03 13:52:04 +00:00
|
|
|
}
|
2021-07-18 13:40:19 +00:00
|
|
|
|
2022-05-19 14:56:42 +00:00
|
|
|
// objectPutAndHash prepare auth parameters and invoke neofs.CreateObject.
|
|
|
|
// Returns object ID and payload sha256 hash.
|
2022-06-06 11:09:09 +00:00
|
|
|
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (*oid.ID, []byte, error) {
|
2022-06-01 17:35:20 +00:00
|
|
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
2022-05-19 14:56:42 +00:00
|
|
|
hash := sha256.New()
|
|
|
|
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
|
|
|
hash.Write(buf)
|
|
|
|
})
|
|
|
|
id, err := n.neoFS.CreateObject(ctx, prm)
|
|
|
|
return id, hash.Sum(nil), n.transformNeofsError(ctx, err)
|
|
|
|
}
|
|
|
|
|
2021-07-18 13:40:19 +00:00
|
|
|
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
|
|
|
|
func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) {
|
2022-06-10 11:57:41 +00:00
|
|
|
var result ListObjectsInfoV1
|
2021-07-18 13:40:19 +00:00
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
prm := allObjectParams{
|
|
|
|
Bucket: p.BktInfo,
|
|
|
|
Delimiter: p.Delimiter,
|
|
|
|
Prefix: p.Prefix,
|
|
|
|
MaxKeys: p.MaxKeys,
|
|
|
|
Marker: p.Marker,
|
2021-07-18 13:40:19 +00:00
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
objects, next, err := n.getLatestObjectsVersions(ctx, prm)
|
|
|
|
if err != nil {
|
2021-07-22 08:57:37 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
if next != nil {
|
2021-07-18 13:40:19 +00:00
|
|
|
result.IsTruncated = true
|
2022-06-10 11:57:41 +00:00
|
|
|
result.NextMarker = objects[len(objects)-1].Name
|
2021-07-18 13:40:19 +00:00
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
result.Prefixes, result.Objects = triageObjects(objects)
|
2021-07-18 13:40:19 +00:00
|
|
|
|
|
|
|
return &result, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ListObjectsV2 returns objects in a bucket for requests of Version 2.
|
|
|
|
func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) {
|
2022-06-10 11:57:41 +00:00
|
|
|
var result ListObjectsInfoV2
|
2021-07-18 13:40:19 +00:00
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
prm := allObjectParams{
|
|
|
|
Bucket: p.BktInfo,
|
|
|
|
Delimiter: p.Delimiter,
|
|
|
|
Prefix: p.Prefix,
|
|
|
|
MaxKeys: p.MaxKeys,
|
|
|
|
Marker: p.StartAfter,
|
|
|
|
ContinuationToken: p.ContinuationToken,
|
2021-07-18 13:40:19 +00:00
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
objects, next, err := n.getLatestObjectsVersions(ctx, prm)
|
|
|
|
if err != nil {
|
2021-07-22 08:57:37 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
if next != nil {
|
2021-07-18 13:40:19 +00:00
|
|
|
result.IsTruncated = true
|
2022-06-10 11:57:41 +00:00
|
|
|
result.NextContinuationToken = next.ID.EncodeToString()
|
2021-07-18 13:40:19 +00:00
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
result.Prefixes, result.Objects = triageObjects(objects)
|
2021-07-29 08:14:14 +00:00
|
|
|
|
2021-07-18 13:40:19 +00:00
|
|
|
return &result, nil
|
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams) ([]*data.ObjectInfo, *data.ObjectInfo, error) {
|
|
|
|
if p.MaxKeys == 0 {
|
|
|
|
return nil, nil, nil
|
2021-07-18 13:40:19 +00:00
|
|
|
}
|
|
|
|
|
2022-05-20 08:26:35 +00:00
|
|
|
var err error
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
cacheKey := cache.CreateObjectsListCacheKey(&p.Bucket.CID, p.Prefix, true)
|
|
|
|
nodeVersions := n.listsCache.GetVersions(cacheKey)
|
2022-05-20 08:26:35 +00:00
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
if nodeVersions == nil {
|
|
|
|
nodeVersions, err = n.treeService.GetLatestVersionsByPrefix(ctx, &p.Bucket.CID, p.Prefix)
|
2022-05-20 08:26:35 +00:00
|
|
|
if err != nil {
|
2022-06-10 11:57:41 +00:00
|
|
|
return nil, nil, err
|
2022-05-20 08:26:35 +00:00
|
|
|
}
|
2022-06-10 11:57:41 +00:00
|
|
|
if err = n.listsCache.PutVersions(cacheKey, nodeVersions); err != nil {
|
2022-05-20 08:26:35 +00:00
|
|
|
n.log.Error("couldn't cache list of objects", zap.Error(err))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
sort.Slice(nodeVersions, func(i, j int) bool {
|
|
|
|
return nodeVersions[i].FilePath < nodeVersions[j].FilePath
|
|
|
|
})
|
|
|
|
|
|
|
|
existed := make(map[string]struct{}, len(nodeVersions)) // to squash the same directories
|
|
|
|
objects := make([]*data.ObjectInfo, 0, p.MaxKeys)
|
|
|
|
|
|
|
|
for _, node := range nodeVersions {
|
|
|
|
if shouldSkip(node, p, existed) {
|
2022-05-20 08:26:35 +00:00
|
|
|
continue
|
|
|
|
}
|
2022-06-10 11:57:41 +00:00
|
|
|
|
|
|
|
if len(objects) == p.MaxKeys {
|
|
|
|
return objects, &data.ObjectInfo{ID: node.OID, Name: node.FilePath}, nil
|
2022-05-20 08:26:35 +00:00
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
if obj := n.objectFromObjectsCacheOrNeoFS(ctx, p.Bucket, node.OID); obj != nil {
|
|
|
|
if oi := objectInfoFromMeta(p.Bucket, obj, p.Prefix, p.Delimiter); oi != nil {
|
|
|
|
objects = append(objects, oi)
|
|
|
|
}
|
|
|
|
}
|
2022-05-20 08:26:35 +00:00
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
return objects, nil, nil
|
2022-05-20 08:26:35 +00:00
|
|
|
}
|
|
|
|
|
2022-05-20 15:02:00 +00:00
|
|
|
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo, prefix, delimiter string) (map[string][]*data.ExtendedObjectInfo, error) {
|
2021-09-01 16:10:31 +00:00
|
|
|
var err error
|
|
|
|
|
2022-05-20 08:26:35 +00:00
|
|
|
cacheKey := cache.CreateObjectsListCacheKey(&bkt.CID, prefix, false)
|
2022-05-20 15:02:00 +00:00
|
|
|
nodeVersions := n.listsCache.GetVersions(cacheKey)
|
2021-09-01 16:10:31 +00:00
|
|
|
|
2022-05-20 15:02:00 +00:00
|
|
|
if nodeVersions == nil {
|
|
|
|
nodeVersions, err = n.treeService.GetAllVersionsByPrefix(ctx, &bkt.CID, prefix)
|
2021-09-01 16:10:31 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-05-20 15:02:00 +00:00
|
|
|
if err = n.listsCache.PutVersions(cacheKey, nodeVersions); err != nil {
|
2021-09-01 16:10:31 +00:00
|
|
|
n.log.Error("couldn't cache list of objects", zap.Error(err))
|
|
|
|
}
|
2021-08-18 13:48:58 +00:00
|
|
|
}
|
2021-07-18 13:40:19 +00:00
|
|
|
|
2022-05-20 15:02:00 +00:00
|
|
|
versions := make(map[string][]*data.ExtendedObjectInfo, len(nodeVersions))
|
2021-09-01 16:10:31 +00:00
|
|
|
|
2022-05-20 15:02:00 +00:00
|
|
|
for _, nodeVersion := range nodeVersions {
|
|
|
|
oi := &data.ObjectInfo{}
|
|
|
|
|
|
|
|
if nodeVersion.DeleteMarker != nil { // delete marker does not match any object in NeoFS
|
|
|
|
oi.ID = nodeVersion.OID
|
2022-06-10 11:57:41 +00:00
|
|
|
oi.Name = nodeVersion.FilePath
|
2022-05-20 15:02:00 +00:00
|
|
|
oi.Owner = nodeVersion.DeleteMarker.Owner
|
|
|
|
oi.Created = nodeVersion.DeleteMarker.Created
|
|
|
|
oi.IsDeleteMarker = true
|
|
|
|
} else {
|
|
|
|
obj := n.objectFromObjectsCacheOrNeoFS(ctx, bkt, nodeVersion.OID)
|
|
|
|
if obj == nil {
|
2021-07-18 13:40:19 +00:00
|
|
|
continue
|
|
|
|
}
|
2022-05-20 15:02:00 +00:00
|
|
|
oi = objectInfoFromMeta(bkt, obj, prefix, delimiter)
|
|
|
|
if oi == nil {
|
|
|
|
continue
|
2021-08-13 14:13:14 +00:00
|
|
|
}
|
|
|
|
}
|
2022-05-20 15:02:00 +00:00
|
|
|
|
|
|
|
eoi := &data.ExtendedObjectInfo{
|
|
|
|
ObjectInfo: oi,
|
|
|
|
NodeVersion: nodeVersion,
|
|
|
|
}
|
|
|
|
|
|
|
|
objVersions, ok := versions[oi.Name]
|
|
|
|
if !ok {
|
|
|
|
objVersions = []*data.ExtendedObjectInfo{eoi}
|
|
|
|
} else if !oi.IsDir {
|
|
|
|
objVersions = append(objVersions, eoi)
|
|
|
|
}
|
|
|
|
versions[oi.Name] = objVersions
|
2021-08-13 14:13:14 +00:00
|
|
|
}
|
2021-07-18 13:40:19 +00:00
|
|
|
|
2021-08-18 13:48:58 +00:00
|
|
|
return versions, nil
|
2021-07-18 13:40:19 +00:00
|
|
|
}
|
2021-07-28 13:28:05 +00:00
|
|
|
|
2022-01-20 14:36:21 +00:00
|
|
|
func IsSystemHeader(key string) bool {
|
|
|
|
return strings.HasPrefix(key, "S3-")
|
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
func shouldSkip(node *data.NodeVersion, p allObjectParams, existed map[string]struct{}) bool {
|
|
|
|
filepath := node.FilePath
|
|
|
|
if len(p.Delimiter) > 0 {
|
|
|
|
tail := strings.TrimPrefix(filepath, p.Prefix)
|
|
|
|
index := strings.Index(tail, p.Delimiter)
|
|
|
|
if index >= 0 {
|
|
|
|
filepath = p.Prefix + tail[:index+1]
|
2021-07-28 13:28:05 +00:00
|
|
|
}
|
|
|
|
}
|
2022-06-10 11:57:41 +00:00
|
|
|
if _, ok := existed[filepath]; ok {
|
|
|
|
return true
|
|
|
|
}
|
2021-07-29 08:14:14 +00:00
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
if filepath <= p.Marker {
|
|
|
|
return true
|
2021-07-29 08:14:14 +00:00
|
|
|
}
|
2022-06-10 11:57:41 +00:00
|
|
|
|
|
|
|
if p.ContinuationToken != "" {
|
|
|
|
if _, ok := existed[continuationToken]; !ok {
|
|
|
|
if p.ContinuationToken != node.OID.EncodeToString() {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
existed[continuationToken] = struct{}{}
|
2021-07-29 08:14:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-06-10 11:57:41 +00:00
|
|
|
existed[filepath] = struct{}{}
|
|
|
|
return false
|
2021-07-29 08:14:14 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 06:56:56 +00:00
|
|
|
func triageObjects(allObjects []*data.ObjectInfo) (prefixes []string, objects []*data.ObjectInfo) {
|
2021-07-29 08:14:14 +00:00
|
|
|
for _, ov := range allObjects {
|
2021-08-27 21:33:50 +00:00
|
|
|
if ov.IsDir {
|
2021-07-29 08:14:14 +00:00
|
|
|
prefixes = append(prefixes, ov.Name)
|
|
|
|
} else {
|
|
|
|
objects = append(objects, ov)
|
2021-07-28 13:28:05 +00:00
|
|
|
}
|
|
|
|
}
|
2021-07-29 08:14:14 +00:00
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2022-05-31 15:03:58 +00:00
|
|
|
func triageExtendedObjects(allObjects []*data.ExtendedObjectInfo) (prefixes []string, objects []*data.ExtendedObjectInfo) {
|
|
|
|
for _, ov := range allObjects {
|
|
|
|
if ov.ObjectInfo.IsDir {
|
|
|
|
prefixes = append(prefixes, ov.ObjectInfo.Name)
|
|
|
|
} else {
|
|
|
|
objects = append(objects, ov)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2021-09-10 06:56:56 +00:00
|
|
|
func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *data.BucketInfo) bool {
|
2022-03-01 15:07:15 +00:00
|
|
|
settings, err := n.GetBucketSettings(ctx, bktInfo)
|
2021-08-10 08:58:40 +00:00
|
|
|
if err != nil {
|
|
|
|
n.log.Warn("couldn't get versioning settings object", zap.Error(err))
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
return settings.VersioningEnabled
|
|
|
|
}
|
2021-09-01 16:10:31 +00:00
|
|
|
|
2022-06-01 17:35:20 +00:00
|
|
|
func (n *layer) objectFromObjectsCacheOrNeoFS(ctx context.Context, bktInfo *data.BucketInfo, obj oid.ID) *object.Object {
|
2021-09-01 16:10:31 +00:00
|
|
|
var (
|
|
|
|
err error
|
2022-06-01 17:35:20 +00:00
|
|
|
meta = n.objCache.Get(newAddress(bktInfo.CID, obj))
|
2021-09-01 16:10:31 +00:00
|
|
|
)
|
|
|
|
if meta == nil {
|
2022-06-01 17:35:20 +00:00
|
|
|
meta, err = n.objectHead(ctx, bktInfo, obj)
|
2021-09-01 16:10:31 +00:00
|
|
|
if err != nil {
|
|
|
|
n.log.Warn("could not fetch object meta", zap.Error(err))
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
if err = n.objCache.Put(*meta); err != nil {
|
|
|
|
n.log.Error("couldn't cache an object", zap.Error(err))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return meta
|
|
|
|
}
|
2022-01-21 12:06:39 +00:00
|
|
|
|
|
|
|
func (n *layer) transformNeofsError(ctx context.Context, err error) error {
|
|
|
|
if err == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-06-06 11:09:09 +00:00
|
|
|
if errors.Is(err, ErrAccessDenied) {
|
2022-01-21 12:06:39 +00:00
|
|
|
n.log.Debug("error was transformed", zap.String("request_id", api.GetRequestID(ctx)), zap.Error(err))
|
|
|
|
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
|
|
|
|
}
|
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
2022-05-19 14:56:42 +00:00
|
|
|
|
|
|
|
func wrapReader(input io.Reader, bufSize int, f func(buf []byte)) io.Reader {
|
2022-05-25 15:59:36 +00:00
|
|
|
if input == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-05-19 14:56:42 +00:00
|
|
|
r, w := io.Pipe()
|
|
|
|
go func() {
|
|
|
|
var buf = make([]byte, bufSize)
|
|
|
|
for {
|
|
|
|
n, err := input.Read(buf)
|
|
|
|
if n > 0 {
|
|
|
|
f(buf[:n])
|
|
|
|
_, _ = w.Write(buf[:n]) // ignore error, input is not ReadCloser
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
_ = w.CloseWithError(err)
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
return r
|
|
|
|
}
|