pogpp/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#451] Handle lock objects using tree service

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-05-26 16:11:14 +03:00 committed by Alex Vanin
parent bc000f1bc4
commit dd534e8738
23 changed files with 488 additions and 520 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
api/cache/system.go vendored
View file

@ -56,6 +56,21 @@ func (o *SystemCache) GetObject(key string) *data.ObjectInfo {
return result
}
// GetLockInfo returns a cached object.
func (o *SystemCache) GetLockInfo(key string) *data.LockInfo {
entry, err := o.cache.Get(key)
if err != nil {
return nil
}
result, ok := entry.(*data.LockInfo)
if !ok {
return nil
}
return result
}
func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
entry, err := o.cache.Get(key)
if err != nil {
@ -124,6 +139,11 @@ func (o *SystemCache) PutObject(key string, obj *data.ObjectInfo) error {
return o.cache.Set(key, obj)
}
// PutLockInfo puts an object to cache.
func (o *SystemCache) PutLockInfo(key string, lockInfo *data.LockInfo) error {
return o.cache.Set(key, lockInfo)
}
func (o *SystemCache) PutCORS(key string, obj *data.CORSConfiguration) error {
return o.cache.Set(key, obj)
}

18
api/data/locking.go
View file

@ -3,8 +3,6 @@ package data
import (
"encoding/xml"
"time"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
type (
@ -36,9 +34,17 @@ type (
}
ObjectLock struct {
Until time.Time
LegalHold bool
IsCompliance bool
Objects []oid.ID
LegalHold *LegalHoldLock
Retention *RetentionLock
}
LegalHoldLock struct {
Enabled bool
}
RetentionLock struct {
Until time.Time
IsCompliance bool
ByPassedGovernance bool
}
)

9
api/data/tree.go
View file

@ -62,3 +62,12 @@ type PartInfo struct {
Number int
OID oid.ID
}
// LockInfo is lock information to create appropriate tree node.
type LockInfo struct {
ID uint64
LegalHoldOID *oid.ID
RetentionOID *oid.ID
UntilDate string
IsCompliance bool
}

3
api/handler/delete.go
View file

@ -4,6 +4,7 @@ import (
"encoding/xml"
"net/http"
"strconv"
"strings"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
@ -144,7 +145,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
func isErrObjectLocked(err error) bool {
switch err.(type) {
default:
return false
return strings.Contains(err.Error(), "object is locked")
case apistatus.ObjectLocked,
*apistatus.ObjectLocked:
return true

8
api/handler/get.go
View file

@ -138,10 +138,10 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
t := &data.ObjectTaggingInfo{
CnrID: &info.CID,
ObjName: info.Name,
VersionID: info.Version(),
t := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: info.Name,
VersionID: info.Version(),
}
tagSet, err := h.obj.GetObjectTagging(r.Context(), t)

39
api/handler/head.go
View file

@ -62,10 +62,10 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
t := &data.ObjectTaggingInfo{
CnrID: &info.CID,
ObjName: info.Name,
VersionID: info.Version(),
t := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: info.Name,
VersionID: info.Version(),
}
tagSet, err := h.obj.GetObjectTagging(r.Context(), t)
@ -119,25 +119,28 @@ func (h *handler) setLockingHeaders(ctx context.Context, bktInfo *data.BucketInf
}
legalHold := &data.LegalHold{Status: legalHoldOff}
legalHoldInfo, err := h.obj.HeadSystemObject(ctx, bktInfo, objInfo.LegalHoldObject())
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
return err
}
if legalHoldInfo != nil {
legalHold.Status = legalHoldOn
}
retention := &data.Retention{Mode: governanceMode}
retentionInfo, err := h.obj.HeadSystemObject(ctx, bktInfo, objInfo.RetentionObject())
p := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objInfo.Name,
VersionID: objInfo.Version(),
}
lockInfo, err := h.obj.GetLockInfo(ctx, p)
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
return err
}
if retentionInfo != nil {
retention.RetainUntilDate = retentionInfo.Headers[layer.AttributeRetainUntil]
if retentionInfo.Headers[layer.AttributeComplianceMode] != "" {
retention.Mode = complianceMode
if lockInfo != nil {
if lockInfo.LegalHoldOID != nil {
legalHold.Status = legalHoldOn
}
if lockInfo.RetentionOID != nil {
retention.RetainUntilDate = lockInfo.UntilDate
if lockInfo.IsCompliance {
retention.Mode = complianceMode
}
}
}

162
api/handler/locking.go
View file

@ -11,7 +11,6 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/data"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
const (
@ -130,46 +129,21 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
return
}
p := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
p := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
lock := &data.ObjectLock{
LegalHold: &data.LegalHoldLock{
Enabled: legalHold.Status == legalHoldOn,
},
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), p)
if err != nil {
h.logAndSendError(w, "could not get object info", reqInfo, err)
if err = h.obj.PutLockInfo(r.Context(), p, lock); err != nil {
h.logAndSendError(w, "couldn't head put legal hold", reqInfo, err)
return
}
lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject())
if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
return
}
if lockInfo == nil && legalHold.Status == legalHoldOff ||
lockInfo != nil && legalHold.Status == legalHoldOn {
return
}
if lockInfo != nil {
if err = h.obj.DeleteSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject()); err != nil {
h.logAndSendError(w, "couldn't delete legal hold", reqInfo, err)
return
}
} else {
ps := &layer.PutSystemObjectParams{
BktInfo: bktInfo,
ObjName: objInfo.LegalHoldObject(),
Lock: &data.ObjectLock{LegalHold: true, Objects: []oid.ID{objInfo.ID}},
Metadata: make(map[string]string),
}
if _, err = h.obj.PutSystemObject(r.Context(), ps); err != nil {
h.logAndSendError(w, "couldn't put legal hold", reqInfo, err)
return
}
}
}
func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
@ -187,26 +161,20 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
return
}
p := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
p := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), p)
lockInfo, err := h.obj.GetLockInfo(r.Context(), p)
if err != nil {
h.logAndSendError(w, "could not get object info", reqInfo, err)
return
}
lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject())
if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
return
}
legalHold := &data.LegalHold{Status: legalHoldOff}
if lockInfo != nil {
if lockInfo.LegalHoldOID != nil {
legalHold.Status = legalHoldOn
}
@ -241,37 +209,13 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
return
}
p := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
p := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), p)
if err != nil {
h.logAndSendError(w, "could not get object info", reqInfo, err)
return
}
lock.Objects = append(lock.Objects, objInfo.ID)
lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.RetentionObject())
if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
return
}
if err = checkLockInfo(lockInfo, r.Header); err != nil {
h.logAndSendError(w, "couldn't change lock mode", reqInfo, err)
return
}
ps := &layer.PutSystemObjectParams{
BktInfo: bktInfo,
ObjName: objInfo.RetentionObject(),
Lock: lock,
Metadata: make(map[string]string),
}
if _, err = h.obj.PutSystemObject(r.Context(), ps); err != nil {
if err = h.obj.PutLockInfo(r.Context(), p, lock); err != nil {
h.logAndSendError(w, "couldn't put legal hold", reqInfo, err)
return
}
@ -308,29 +252,28 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
return
}
p := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
p := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), p)
if err != nil {
h.logAndSendError(w, "could not get object info", reqInfo, err)
return
}
lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.RetentionObject())
lockInfo, err := h.obj.GetLockInfo(r.Context(), p)
if err != nil {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
return
}
if lockInfo.RetentionOID == nil {
h.logAndSendError(w, "retention lock isn't set", reqInfo, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey))
return
}
retention := &data.Retention{
Mode: governanceMode,
RetainUntilDate: lockInfo.Headers[layer.AttributeRetainUntil],
RetainUntilDate: lockInfo.UntilDate,
}
if lockInfo.Headers[layer.AttributeComplianceMode] != "" {
if lockInfo.IsCompliance {
retention.Mode = complianceMode
}
@ -379,21 +322,28 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
}
if defaultConfig.Rule != nil && defaultConfig.Rule.DefaultRetention != nil {
retention := &data.RetentionLock{}
defaultRetention := defaultConfig.Rule.DefaultRetention
objectLock.IsCompliance = defaultRetention.Mode == complianceMode
retention.IsCompliance = defaultRetention.Mode == complianceMode
now := time.Now()
if defaultRetention.Days != 0 {
objectLock.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration)
retention.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration)
} else {
objectLock.Until = now.Add(time.Duration(defaultRetention.Years) * yearDuration)
retention.Until = now.Add(time.Duration(defaultRetention.Years) * yearDuration)
}
objectLock.Retention = retention
}
objectLock.LegalHold = header.Get(api.AmzObjectLockLegalHold) == legalHoldOn
if header.Get(api.AmzObjectLockLegalHold) == legalHoldOn {
objectLock.LegalHold = &data.LegalHoldLock{Enabled: true}
}
mode := header.Get(api.AmzObjectLockMode)
if mode != "" {
objectLock.IsCompliance = mode == complianceMode
if objectLock.Retention == nil {
objectLock.Retention = &data.RetentionLock{}
}
objectLock.Retention.IsCompliance = mode == complianceMode
}
until := header.Get(api.AmzObjectLockRetainUntilDate)
@ -402,7 +352,10 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
if err != nil {
return nil, fmt.Errorf("invalid header %s: '%s'", api.AmzObjectLockRetainUntilDate, until)
}
objectLock.Until = retentionDate
if objectLock.Retention == nil {
objectLock.Retention = &data.RetentionLock{}
}
objectLock.Retention.Until = retentionDate
}
return objectLock, nil
@ -424,9 +377,20 @@ func formObjectLockFromRetention(retention *data.Retention, header http.Header)
return nil, fmt.Errorf("couldn't parse retain until date: %s", retention.RetainUntilDate)
}
var bypass bool
if bypassStr := header.Get(api.AmzBypassGovernanceRetention); len(bypassStr) > 0 {
bypass, err = strconv.ParseBool(bypassStr)
if err != nil {
return nil, fmt.Errorf("couldn't parse bypass governance header: %w", err)
}
}
lock := &data.ObjectLock{
Until: retentionDate,
IsCompliance: retention.Mode == complianceMode,
Retention: &data.RetentionLock{
Until: retentionDate,
IsCompliance: retention.Mode == complianceMode,
ByPassedGovernance: bypass,
},
}
return lock, nil

23
api/handler/locking_test.go
View file

@ -33,14 +33,17 @@ func TestFormObjectLock(t *testing.T) {
bktInfo: &data.BucketInfo{ObjectLockEnabled: true},
config: &data.ObjectLockConfiguration{Rule: &data.ObjectLockRule{
DefaultRetention: &data.DefaultRetention{Mode: complianceMode, Days: 1}}},
expectedLock: &data.ObjectLock{IsCompliance: true, Until: time.Now().Add(24 * time.Hour)},
expectedLock: &data.ObjectLock{Retention: &data.RetentionLock{
IsCompliance: true,
Until: time.Now().Add(24 * time.Hour)}},
},
{
name: "default years",
bktInfo: &data.BucketInfo{ObjectLockEnabled: true},
config: &data.ObjectLockConfiguration{Rule: &data.ObjectLockRule{
DefaultRetention: &data.DefaultRetention{Mode: governanceMode, Years: 1}}},
expectedLock: &data.ObjectLock{Until: time.Now().Add(365 * 24 * time.Hour)},
expectedLock: &data.ObjectLock{Retention: &data.RetentionLock{
Until: time.Now().Add(365 * 24 * time.Hour)}},
},
{
name: "basic override",
@ -51,7 +54,9 @@ func TestFormObjectLock(t *testing.T) {
api.AmzObjectLockMode: {governanceMode},
api.AmzObjectLockLegalHold: {legalHoldOn},
},
expectedLock: &data.ObjectLock{Until: time.Now(), LegalHold: true},
expectedLock: &data.ObjectLock{
LegalHold: &data.LegalHoldLock{Enabled: true},
Retention: &data.RetentionLock{Until: time.Now()}},
},
{
name: "lock disabled error",
@ -95,7 +100,9 @@ func TestFormObjectLockFromRetention(t *testing.T) {
Mode: complianceMode,
RetainUntilDate: time.Now().Format(time.RFC3339),
},
expectedLock: &data.ObjectLock{Until: time.Now(), IsCompliance: true},
expectedLock: &data.ObjectLock{Retention: &data.RetentionLock{
Until: time.Now(),
IsCompliance: true}},
},
{
name: "basic governance",
@ -106,7 +113,7 @@ func TestFormObjectLockFromRetention(t *testing.T) {
header: map[string][]string{
api.AmzBypassGovernanceRetention: {strconv.FormatBool(true)},
},
expectedLock: &data.ObjectLock{Until: time.Now()},
expectedLock: &data.ObjectLock{Retention: &data.RetentionLock{Until: time.Now()}},
},
{
name: "error invalid mode",
@ -140,8 +147,10 @@ func TestFormObjectLockFromRetention(t *testing.T) {
func assertObjectLocks(t *testing.T, expected, actual *data.ObjectLock) {
require.Equal(t, expected.LegalHold, actual.LegalHold)
require.Equal(t, expected.IsCompliance, actual.IsCompliance)
require.InDelta(t, expected.Until.Unix(), actual.Until.Unix(), 1)
if expected.Retention != nil {
require.Equal(t, expected.Retention.IsCompliance, actual.Retention.IsCompliance)
require.InDelta(t, expected.Retention.Until.Unix(), actual.Retention.Until.Unix(), 1)
}
}
func TestCheckLockObject(t *testing.T) {

8
api/handler/multipart_upload.go
View file

@ -370,10 +370,10 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
}
if len(uploadData.TagSet) != 0 {
t := &data.ObjectTaggingInfo{
CnrID: &bktInfo.CID,
ObjName: objInfo.Name,
VersionID: objInfo.Version(),
t := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: objInfo.Name,
VersionID: objInfo.Version(),
}
if err = h.obj.PutObjectTagging(r.Context(), t, uploadData.TagSet); err != nil {
h.logAndSendError(w, "could not put tagging file of completed multipart upload", reqInfo, err, additional...)

16
api/handler/put.go
View file

@ -252,10 +252,10 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
}
}
t := &data.ObjectTaggingInfo{
CnrID: &info.CID,
ObjName: info.Name,
VersionID: info.Version(),
t := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: info.Name,
VersionID: info.Version(),
}
if tagSet != nil {
if err = h.obj.PutObjectTagging(r.Context(), t, tagSet); err != nil {
@ -379,10 +379,10 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
}
}
t := &data.ObjectTaggingInfo{
CnrID: &info.CID,
ObjName: info.Name,
VersionID: info.Version(),
t := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: info.Name,
VersionID: info.Version(),
}
if tagSet != nil {

25
api/handler/tagging.go
View file

@ -11,6 +11,7 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
)
@ -37,10 +38,10 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
p := &data.ObjectTaggingInfo{
CnrID: &bktInfo.CID,
ObjName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get("versionId"),
p := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get("versionId"),
}
if err = h.obj.PutObjectTagging(r.Context(), p, tagSet); err != nil {
@ -73,10 +74,10 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
return
}
p := &data.ObjectTaggingInfo{
CnrID: &bktInfo.CID,
ObjName: reqInfo.ObjectName,
VersionID: versionID,
p := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: versionID,
}
tagSet, err := h.obj.GetObjectTagging(r.Context(), p)
@ -100,10 +101,10 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
return
}
p := &data.ObjectTaggingInfo{
CnrID: &bktInfo.CID,
ObjName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get("versionId"),
p := &layer.ObjectVersion{
BktInfo: bktInfo,
ObjectName: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get("versionId"),
}
if err = h.obj.DeleteObjectTagging(r.Context(), p); err != nil {

18
api/layer/cors.go
View file

@ -36,21 +36,19 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
return err
}
s := &PutSystemObjectParams{
BktInfo: p.BktInfo,
ObjName: p.BktInfo.CORSObjectName(),
Metadata: map[string]string{},
Prefix: "",
Reader: &buf,
Size: int64(buf.Len()),
prm := PrmObjectCreate{
Container: p.BktInfo.CID,
Creator: p.BktInfo.Owner,
Payload: p.Reader,
Filename: p.BktInfo.CORSObjectName(),
}
obj, err := n.putSystemObjectIntoNeoFS(ctx, s)
objID, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
if err != nil {
return fmt.Errorf("put system object: %w", err)
}
objIDToDelete, err := n.treeService.PutBucketCORS(ctx, &p.BktInfo.CID, &obj.ID)
objIDToDelete, err := n.treeService.PutBucketCORS(ctx, &p.BktInfo.CID, objID)
if err != nil {
return err
}
@ -64,7 +62,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
}
}
if err := n.systemCache.PutCORS(systemObjectKey(p.BktInfo, s.ObjName), cors); err != nil {
if err := n.systemCache.PutCORS(systemObjectKey(p.BktInfo, prm.Filename), cors); err != nil {
n.log.Error("couldn't cache system object", zap.Error(err))
}

35
api/layer/layer.go
View file

@ -90,6 +90,13 @@ type (
VersionID string
}
// ObjectVersion stores object version info.
ObjectVersion struct {
BktInfo *data.BucketInfo
ObjectName string
VersionID string
}
// RangeParams stores range header request parameters.
RangeParams struct {
Start uint64
@ -208,19 +215,20 @@ type (
DeleteBucket(ctx context.Context, p *DeleteBucketParams) error
GetObject(ctx context.Context, p *GetObjectParams) error
HeadSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) (*data.ObjectInfo, error)
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error)
GetLockInfo(ctx context.Context, obj *ObjectVersion) (*data.LockInfo, error)
PutLockInfo(ctx context.Context, p *ObjectVersion, lock *data.ObjectLock) error
GetBucketTagging(ctx context.Context, cnrID *cid.ID) (map[string]string, error)
PutBucketTagging(ctx context.Context, cnrID *cid.ID, tagSet map[string]string) error
DeleteBucketTagging(ctx context.Context, cnrID *cid.ID) error
GetObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) (map[string]string, error)
PutObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo, tagSet map[string]string) error
DeleteObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) error
GetObjectTagging(ctx context.Context, p *ObjectVersion) (map[string]string, error)
PutObjectTagging(ctx context.Context, p *ObjectVersion, tagSet map[string]string) error
DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error
PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error)
PutSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error)
CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ObjectInfo, error)
@ -245,8 +253,7 @@ type (
)
const (
tagPrefix = "S3-Tag-"
tagEmptyMark = "\\"
tagPrefix = "S3-Tag-"
)
func (t *VersionedObject) String() string {
@ -520,17 +527,17 @@ func (n *layer) removeVersionIfFound(ctx context.Context, bkt *data.BucketInfo,
deleteMarkVersion = obj.VersionID
}
if err := n.treeService.RemoveVersion(ctx, &bkt.CID, version.ID); err != nil {
return deleteMarkVersion, err
}
if err := n.objectDelete(ctx, bkt, version.OID); err != nil {
return deleteMarkVersion, err
}
if err := n.treeService.RemoveVersion(ctx, &bkt.CID, version.ID); err != nil {
return deleteMarkVersion, err
}
p := &data.ObjectTaggingInfo{
CnrID: &bkt.CID,
ObjName: obj.Name,
VersionID: version.OID.EncodeToString(),
p := &ObjectVersion{
BktInfo: bkt,
ObjectName: obj.Name,
VersionID: version.OID.EncodeToString(),
}
return deleteMarkVersion, n.DeleteObjectTagging(ctx, p)
}

26
api/layer/locking_test.go
View file

@ -5,7 +5,6 @@ import (
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/stretchr/testify/require"
)
@ -19,18 +18,25 @@ func TestObjectLockAttributes(t *testing.T) {
obj := tc.putObject([]byte("content obj1 v1"))
_, err = tc.layer.PutSystemObject(tc.ctx, &PutSystemObjectParams{
BktInfo: tc.bktInfo,
ObjName: obj.RetentionObject(),
Metadata: make(map[string]string),
Lock: &data.ObjectLock{
Until: time.Now(),
Objects: []oid.ID{obj.ID},
p := &ObjectVersion{
BktInfo: tc.bktInfo,
ObjectName: obj.Name,
VersionID: obj.Version(),
}
lock := &data.ObjectLock{
Retention: &data.RetentionLock{
Until: time.Now(),
},
})
}
err = tc.layer.PutLockInfo(tc.ctx, p, lock)
require.NoError(t, err)
lockObj := tc.getSystemObject(obj.RetentionObject())
foundLock, err := tc.layer.GetLockInfo(tc.ctx, p)
require.NoError(t, err)
lockObj := tc.getObjectByID(*foundLock.RetentionOID)
require.NotNil(t, lockObj)
expEpoch := false

15
api/layer/notifications.go
View file

@ -26,20 +26,19 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
sysName := p.BktInfo.NotificationConfigurationObjectName()
s := &PutSystemObjectParams{
BktInfo: p.BktInfo,
ObjName: sysName,
Metadata: map[string]string{},
Reader: bytes.NewReader(confXML),
Size: int64(len(confXML)),
prm := PrmObjectCreate{
Container: p.BktInfo.CID,
Creator: p.BktInfo.Owner,
Payload: bytes.NewReader(confXML),
Filename: sysName,
}
obj, err := n.putSystemObjectIntoNeoFS(ctx, s)
objID, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
if err != nil {
return err
}
objIDToDelete, err := n.treeService.PutNotificationConfigurationNode(ctx, &p.BktInfo.CID, &obj.ID)
objIDToDelete, err := n.treeService.PutNotificationConfigurationNode(ctx, &p.BktInfo.CID, objID)
if err != nil {
return err
}

34
api/layer/object.go
View file

@ -220,18 +220,15 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
zap.Error(err))
}
if p.Lock != nil {
objInfo := &data.ObjectInfo{ID: *id, Name: p.Object}
p.Lock.Objects = append(p.Lock.Objects, *id)
if p.Lock.LegalHold {
if err = n.putLockObject(ctx, p.BktInfo, objInfo.LegalHoldObject(), p.Lock); err != nil {
return nil, err
}
if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) {
objVersion := &ObjectVersion{
BktInfo: p.BktInfo,
ObjectName: p.Object,
VersionID: id.EncodeToString(),
}
if !p.Lock.Until.IsZero() {
if err = n.putLockObject(ctx, p.BktInfo, objInfo.RetentionObject(), p.Lock); err != nil {
return nil, err
}
if err = n.PutLockInfo(ctx, objVersion, p.Lock); err != nil {
return nil, err
}
}
@ -266,21 +263,6 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
return objInfo, nil
}
func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objName string, lock *data.ObjectLock) error {
ps := &PutSystemObjectParams{
BktInfo: bktInfo,
ObjName: objName,
Lock: lock,
Metadata: make(map[string]string),
}
if _, err := n.PutSystemObject(ctx, ps); err != nil {
return fmt.Errorf("coudln't add lock for '%s': %w", objName, err)
}
return nil
}
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ObjectInfo, error) {
if addr := n.namesCache.Get(bkt.Name + "/" + objectName); addr != nil {
if headInfo := n.objCache.Get(*addr); headInfo != nil {

250
api/layer/system_object.go
View file

@ -2,7 +2,6 @@ package layer
import (
"context"
"encoding/hex"
"encoding/xml"
errorsStd "errors"
"fmt"
@ -11,56 +10,123 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/internal/misc"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)
const (
AttributeComplianceMode = ".s3-compliance-mode"
AttributeRetainUntil = ".s3-retain-until"
AttributeExpirationEpoch = "__NEOFS__EXPIRATION_EPOCH"
AttributeSysTickEpoch = "__NEOFS__TICK_EPOCH"
AttributeSysTickTopic = "__NEOFS__TICK_TOPIC"
)
func (n *layer) PutSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) {
objInfo, err := n.putSystemObjectIntoNeoFS(ctx, p)
func (n *layer) PutLockInfo(ctx context.Context, objVersion *ObjectVersion, newLock *data.ObjectLock) error {
cnrID := objVersion.BktInfo.CID
versionNode, err := n.getNodeVersion(ctx, objVersion)
if err != nil {
return nil, err
return err
}
if err = n.systemCache.PutObject(systemObjectKey(p.BktInfo, p.ObjName), objInfo); err != nil {
lockInfo, err := n.treeService.GetLock(ctx, &cnrID, versionNode.ID)
if err != nil && !errorsStd.Is(err, ErrNodeNotFound) {
return err
}
if lockInfo == nil {
lockInfo = &data.LockInfo{}
}
if newLock.Retention != nil {
if lockInfo.RetentionOID != nil {
if lockInfo.IsCompliance {
return fmt.Errorf("you cannot change compliance mode")
}
if !newLock.Retention.ByPassedGovernance {
return fmt.Errorf("you cannot bypass governence mode")
}
if len(lockInfo.UntilDate) > 0 {
parsedTime, err := time.Parse(time.RFC3339, lockInfo.UntilDate)
if err != nil {
return fmt.Errorf("couldn't parse time '%s': %w", lockInfo.UntilDate, err)
}
if parsedTime.After(newLock.Retention.Until) {
return fmt.Errorf("you couldn't short the until date")
}
}
}
lock := &data.ObjectLock{Retention: newLock.Retention}
if lockInfo.RetentionOID, err = n.putLockObject(ctx, objVersion.BktInfo, versionNode.OID, lock); err != nil {
return err
}
lockInfo.IsCompliance = newLock.Retention.IsCompliance
lockInfo.UntilDate = newLock.Retention.Until.UTC().Format(time.RFC3339)
}
if newLock.LegalHold != nil {
if newLock.LegalHold.Enabled && lockInfo.LegalHoldOID == nil {
lock := &data.ObjectLock{LegalHold: newLock.LegalHold}
if lockInfo.LegalHoldOID, err = n.putLockObject(ctx, objVersion.BktInfo, versionNode.OID, lock); err != nil {
return err
}
} else if !newLock.LegalHold.Enabled && lockInfo.LegalHoldOID != nil {
if err = n.objectDelete(ctx, objVersion.BktInfo, *lockInfo.LegalHoldOID); err != nil {
return fmt.Errorf("couldn't delete lock object '%s' to remove legal hold: %w", lockInfo.LegalHoldOID.EncodeToString(), err)
}
lockInfo.LegalHoldOID = nil
}
}
if err = n.treeService.PutLock(ctx, &cnrID, versionNode.ID, lockInfo); err != nil {
return fmt.Errorf("couldn't put lock into tree: %w", err)
}
if err = n.systemCache.PutLockInfo(lockObjectKey(objVersion), lockInfo); err != nil {
n.log.Error("couldn't cache system object", zap.Error(err))
}
return objInfo, nil
return nil
}
func (n *layer) HeadSystemObject(ctx context.Context, bkt *data.BucketInfo, objName string) (*data.ObjectInfo, error) {
if objInfo := n.systemCache.GetObject(systemObjectKey(bkt, objName)); objInfo != nil {
return objInfo, nil
func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, lock *data.ObjectLock) (*oid.ID, error) {
prm := PrmObjectCreate{
Container: bktInfo.CID,
Creator: bktInfo.Owner,
Locks: []oid.ID{objID},
}
node, err := n.treeService.GetSystemVersion(ctx, &bkt.CID, objName)
if err != nil {
if errorsStd.Is(err, ErrNodeNotFound) {
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
}
return nil, err
}
meta, err := n.objectHead(ctx, bkt, node.OID)
var err error
prm.Attributes, err = n.attributesFromLock(ctx, lock)
if err != nil {
return nil, err
}
objInfo := objInfoFromMeta(bkt, meta)
if err = n.systemCache.PutObject(systemObjectKey(bkt, objName), objInfo); err != nil {
id, _, err := n.objectPutAndHash(ctx, prm, bktInfo)
return id, err
}
func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) {
if lockInfo := n.systemCache.GetLockInfo(lockObjectKey(objVersion)); lockInfo != nil {
return lockInfo, nil
}
versionNode, err := n.getNodeVersion(ctx, objVersion)
if err != nil {
return nil, err
}
lockInfo, err := n.treeService.GetLock(ctx, &objVersion.BktInfo.CID, versionNode.ID)
if err != nil && !errorsStd.Is(err, ErrNodeNotFound) {
return nil, err
}
if lockInfo == nil {
lockInfo = &data.LockInfo{}
}
if err = n.systemCache.PutLockInfo(lockObjectKey(objVersion), lockInfo); err != nil {
n.log.Error("couldn't cache system object", zap.Error(err))
}
return objInfo, nil
return lockInfo, nil
}
func (n *layer) DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error {
@ -83,97 +149,6 @@ func (n *layer) DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo
return nil
}
func (n *layer) putSystemObjectIntoNeoFS(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) {
prm := PrmObjectCreate{
Container: p.BktInfo.CID,
Creator: p.BktInfo.Owner,
Attributes: make([][2]string, 2, 2+len(p.Metadata)),
Payload: p.Reader,
}
prm.Attributes[0][0], prm.Attributes[0][1] = objectSystemAttributeName, p.ObjName
prm.Attributes[1][0], prm.Attributes[1][1] = attrVersionsIgnore, "true"
for k, v := range p.Metadata {
if !IsSystemHeader(k) {
k = p.Prefix + k
}
if v == "" && p.Prefix == tagPrefix {
v = tagEmptyMark
}
if p.Lock != nil && len(p.Lock.Objects) > 0 {
prm.Locks = p.Lock.Objects
attrs, err := n.attributesFromLock(ctx, p.Lock)
if err != nil {
return nil, fmt.Errorf("get lock attributes: %w", err)
}
prm.Attributes = append(prm.Attributes, attrs...)
}
prm.Attributes = append(prm.Attributes, [2]string{k, v})
}
id, hash, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
if err != nil {
return nil, err
}
newVersion := &data.BaseNodeVersion{OID: *id}
if err = n.treeService.AddSystemVersion(ctx, &p.BktInfo.CID, p.ObjName, newVersion); err != nil {
return nil, fmt.Errorf("couldn't add new verion to tree service: %w", err)
}
currentEpoch, _, err := n.neoFS.TimeToEpoch(ctx, time.Now().Add(time.Minute))
if err != nil {
n.log.Warn("couldn't get creation epoch",
zap.String("bucket", p.BktInfo.Name),
zap.String("object", misc.SanitizeString(p.ObjName)),
zap.Error(err))
}
headers := make(map[string]string, len(p.Metadata))
for _, attr := range prm.Attributes {
headers[attr[0]] = attr[1]
}
return &data.ObjectInfo{
ID: *id,
CID: p.BktInfo.CID,
Owner: p.BktInfo.Owner,
Bucket: p.BktInfo.Name,
Name: p.ObjName,
Created: time.Now(),
CreationEpoch: currentEpoch,
Size: p.Size,
Headers: headers,
HashSum: hex.EncodeToString(hash),
}, nil
}
func (n *layer) getSystemObjectFromNeoFS(ctx context.Context, bkt *data.BucketInfo, objName string) (*object.Object, error) {
versions, err := n.headSystemVersions(ctx, bkt, objName)
if err != nil {
return nil, err
}
objInfo := versions.getLast()
obj, err := n.objectGet(ctx, bkt, objInfo.ID)
if err != nil {
return nil, err
}
if len(obj.Payload()) == 0 {
return nil, errors.GetAPIError(errors.ErrInternalError)
}
return obj, nil
}
func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo, sysName string) (*data.CORSConfiguration, error) {
if cors := n.systemCache.GetCORS(systemObjectKey(bkt, sysName)); cors != nil {
return cors, nil
@ -251,6 +226,11 @@ func systemObjectKey(bktInfo *data.BucketInfo, obj string) string {
return bktInfo.Name + obj
}
func lockObjectKey(objVersion *ObjectVersion) string {
// todo reconsider forming name since versionID can be "null" or ""
return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID
}
func (n *layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
systemKey := systemObjectKey(bktInfo, bktInfo.SettingsObjectName())
if settings := n.systemCache.GetSettings(systemKey); settings != nil {
@ -289,26 +269,24 @@ func (n *layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) err
}
func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) ([][2]string, error) {
var result [][2]string
if !lock.Until.IsZero() {
_, exp, err := n.neoFS.TimeToEpoch(ctx, lock.Until)
if err != nil {
return nil, fmt.Errorf("fetch time to epoch: %w", err)
}
attrs := [][2]string{
{AttributeExpirationEpoch, strconv.FormatUint(exp, 10)},
{AttributeRetainUntil, lock.Until.Format(time.RFC3339)},
}
result = append(result, attrs...)
if lock.IsCompliance {
attrCompliance := [2]string{
AttributeComplianceMode, strconv.FormatBool(true),
}
result = append(result, attrCompliance)
}
if lock.Retention == nil {
return nil, nil
}
_, exp, err := n.neoFS.TimeToEpoch(ctx, lock.Retention.Until)
if err != nil {
return nil, fmt.Errorf("fetch time to epoch: %w", err)
}
result := [][2]string{
{AttributeExpirationEpoch, strconv.FormatUint(exp, 10)},
}
if lock.Retention.IsCompliance {
attrCompliance := [2]string{
AttributeComplianceMode, strconv.FormatBool(true),
}
result = append(result, attrCompliance)
}
return result, nil
}

64
api/layer/tagging.go
View file

@ -3,13 +3,14 @@ package layer
import (
"context"
errorsStd "errors"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"go.uber.org/zap"
)
func (n *layer) GetObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) (map[string]string, error) {
func (n *layer) GetObjectTagging(ctx context.Context, p *ObjectVersion) (map[string]string, error) {
var (
err error
tags map[string]string
@ -19,12 +20,12 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo)
return tags, nil
}
version, err := n.getTaggedObjectVersion(ctx, p)
version, err := n.getNodeVersion(ctx, p)
if err != nil {
return nil, err
}
tags, err = n.treeService.GetObjectTagging(ctx, p.CnrID, version)
tags, err = n.treeService.GetObjectTagging(ctx, &p.BktInfo.CID, version)
if err != nil {
if errorsStd.Is(err, ErrNodeNotFound) {
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
@ -39,13 +40,13 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo)
return tags, nil
}
func (n *layer) PutObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo, tagSet map[string]string) error {
version, err := n.getTaggedObjectVersion(ctx, p)
func (n *layer) PutObjectTagging(ctx context.Context, p *ObjectVersion, tagSet map[string]string) error {
version, err := n.getNodeVersion(ctx, p)
if err != nil {
return err
}
err = n.treeService.PutObjectTagging(ctx, p.CnrID, version, tagSet)
err = n.treeService.PutObjectTagging(ctx, &p.BktInfo.CID, version, tagSet)
if err != nil {
if errorsStd.Is(err, ErrNodeNotFound) {
return errors.GetAPIError(errors.ErrNoSuchKey)
@ -60,13 +61,13 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo,
return nil
}
func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) error {
version, err := n.getTaggedObjectVersion(ctx, p)
func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error {
version, err := n.getNodeVersion(ctx, p)
if err != nil {
return err
}
err = n.treeService.DeleteObjectTagging(ctx, p.CnrID, version)
err = n.treeService.DeleteObjectTagging(ctx, &p.BktInfo.CID, version)
if err != nil {
if errorsStd.Is(err, ErrNodeNotFound) {
return errors.GetAPIError(errors.ErrNoSuchKey)
@ -118,50 +119,41 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, cnrID *cid.ID) error {
return n.treeService.DeleteBucketTagging(ctx, cnrID)
}
func objectTaggingCacheKey(p *data.ObjectTaggingInfo) string {
return ".tagset." + p.CnrID.EncodeToString() + "." + p.ObjName + "." + p.VersionID
func objectTaggingCacheKey(p *ObjectVersion) string {
return ".tagset." + p.BktInfo.CID.EncodeToString() + "." + p.ObjectName + "." + p.VersionID
}
func bucketTaggingCacheKey(cnrID *cid.ID) string {
return ".tagset." + cnrID.EncodeToString()
}
func (n *layer) getTaggedObjectVersion(ctx context.Context, p *data.ObjectTaggingInfo) (*data.NodeVersion, error) {
var (
err error
version *data.NodeVersion
)
func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (*data.NodeVersion, error) {
var err error
var version *data.NodeVersion
if p.VersionID == unversionedObjectVersionID {
if version, err = n.treeService.GetUnversioned(ctx, p.CnrID, p.ObjName); err != nil {
if errorsStd.Is(err, ErrNodeNotFound) {
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
}
return nil, err
}
} else if len(p.VersionID) == 0 {
if version, err = n.treeService.GetLatestVersion(ctx, p.CnrID, p.ObjName); err != nil {
if errorsStd.Is(err, ErrNodeNotFound) {
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
}
return nil, err
}
if objVersion.VersionID == unversionedObjectVersionID {
version, err = n.treeService.GetUnversioned(ctx, &objVersion.BktInfo.CID, objVersion.ObjectName)
} else if len(objVersion.VersionID) == 0 {
version, err = n.treeService.GetLatestVersion(ctx, &objVersion.BktInfo.CID, objVersion.ObjectName)
} else {
versions, err := n.treeService.GetVersions(ctx, p.CnrID, p.ObjName)
if err != nil {
return nil, err
versions, err2 := n.treeService.GetVersions(ctx, &objVersion.BktInfo.CID, objVersion.ObjectName)
if err2 != nil {
return nil, err2
}
for _, v := range versions {
if v.OID.EncodeToString() == p.VersionID {
if v.OID.EncodeToString() == objVersion.VersionID {
version = v
break
}
}
if version == nil {
err = errors.GetAPIError(errors.ErrNoSuchKey)
}
}
if version == nil || version.DeleteMarker != nil {
if err == nil && version.DeleteMarker != nil || errorsStd.Is(err, ErrNodeNotFound) {
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
}
return version, nil
return version, err
}

5
api/layer/tree_service.go
View file

@ -46,9 +46,8 @@ type TreeService interface {
AddVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *data.NodeVersion) error
RemoveVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error
AddSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *data.BaseNodeVersion) error
GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error)
RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error
PutLock(ctx context.Context, cnrID *cid.ID, nodeID uint64, lock *data.LockInfo) error
GetLock(ctx context.Context, cnrID *cid.ID, nodeID uint64) (*data.LockInfo, error)
CreateMultipartUpload(ctx context.Context, cnrID *cid.ID, info *data.MultipartInfo) error
DeleteMultipartUpload(ctx context.Context, cnrID *cid.ID, multipartNodeID uint64) error

59
api/layer/versioning.go
View file

@ -4,7 +4,6 @@ import (
"context"
"math"
"sort"
"strconv"
"strings"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
@ -42,13 +41,12 @@ const (
VersionsDeleteMarkAttr = "S3-Versions-delete-mark"
DelMarkFullObject = "*"
unversionedObjectVersionID = "null"
objectSystemAttributeName = "S3-System-name"
attrVersionsIgnore = "S3-Versions-ignore"
attrSettingsVersioningEnabled = "S3-Settings-Versioning-enabled"
versionsDelAttr = "S3-Versions-del"
versionsAddAttr = "S3-Versions-add"
versionsUnversionedAttr = "S3-Versions-unversioned"
unversionedObjectVersionID = "null"
objectSystemAttributeName = "S3-System-name"
attrVersionsIgnore = "S3-Versions-ignore"
versionsDelAttr = "S3-Versions-del"
versionsAddAttr = "S3-Versions-add"
versionsUnversionedAttr = "S3-Versions-unversioned"
)
func newObjectVersions(name string) *objectVersions {
@ -213,56 +211,11 @@ func (v *objectVersions) existedVersions() []string {
return res
}
func (v *objectVersions) getFiltered(reverse bool) []*data.ObjectInfo {
if len(v.objects) == 0 {
return nil
}
v.sort()
existedVersions := v.existedVersions()
res := make([]*data.ObjectInfo, 0, len(v.objects))
for _, version := range v.objects {
delMark := version.Headers[VersionsDeleteMarkAttr]
if contains(existedVersions, version.Version()) && (delMark == DelMarkFullObject || delMark == "") {
res = append(res, version)
}
}
if reverse {
for i, j := 0, len(res)-1; i < j; i, j = i+1, j-1 {
res[i], res[j] = res[j], res[i]
}
}
return res
}
func (v *objectVersions) getAddHeader() string {
v.sort()
return strings.Join(v.addList, ",")
}
func (n *layer) PutBucketVersioning(ctx context.Context, p *PutSettingsParams) (*data.ObjectInfo, error) {
metadata := map[string]string{
attrSettingsVersioningEnabled: strconv.FormatBool(p.Settings.VersioningEnabled),
}
s := &PutSystemObjectParams{
BktInfo: p.BktInfo,
ObjName: p.BktInfo.SettingsObjectName(),
Metadata: metadata,
Prefix: "",
Reader: nil,
}
return n.PutSystemObject(ctx, s)
}
func (n *layer) GetBucketVersioning(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
return n.GetBucketSettings(ctx, bktInfo)
}
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
var (
allObjects = make([]*data.ObjectInfo, 0, p.MaxKeys)

10
api/layer/versioning_test.go
View file

@ -127,6 +127,16 @@ func (tc *testContext) getSystemObject(objectName string) *object.Object {
return nil
}
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
for _, obj := range tc.testNeoFS.Objects() {
id, _ := obj.ID()
if id.Equals(objID) {
return obj
}
}
return nil
}
type testContext struct {
t *testing.T
ctx context.Context

103
internal/neofs/tree.go
View file

@ -56,6 +56,13 @@ const (
uploadIDKV = "UploadId"
partNumberKV = "Number"
// keys for lock.
isLockKV = "IsLock"
legalHoldOIDKV = "LegalHoldOID"
retentionOIDKV = "RetentionOID"
untilDateKV = "UntilDate"
isComplianceKV = "IsCompliance"
// keys for delete marker nodes.
isDeleteMarkerKV = "IdDeleteMarker"
filePathKV = "FilePath"
@ -339,7 +346,7 @@ func (c *TreeClient) DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.
}
func (c *TreeClient) GetObjectTagging(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion) (map[string]string, error) {
tagNode, err := c.getObjectTaggingNode(ctx, cnrID, objVersion)
tagNode, err := c.getTreeNode(ctx, cnrID, objVersion.ID, isTagKV)
if err != nil {
return nil, err
}
@ -360,7 +367,7 @@ func (c *TreeClient) GetObjectTagging(ctx context.Context, cnrID *cid.ID, objVer
}
func (c *TreeClient) PutObjectTagging(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion, tagSet map[string]string) error {
tagNode, err := c.getObjectTaggingNode(ctx, cnrID, objVersion)
tagNode, err := c.getTreeNode(ctx, cnrID, objVersion.ID, isTagKV)
if err != nil {
return err
}
@ -382,7 +389,7 @@ func (c *TreeClient) PutObjectTagging(ctx context.Context, cnrID *cid.ID, objVer
}
func (c *TreeClient) DeleteObjectTagging(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion) error {
tagNode, err := c.getObjectTaggingNode(ctx, cnrID, objVersion)
tagNode, err := c.getTreeNode(ctx, cnrID, objVersion.ID, isTagKV)
if err != nil {
return err
}
@ -450,26 +457,26 @@ func (c *TreeClient) DeleteBucketTagging(ctx context.Context, cnrID *cid.ID) err
return nil
}
func (c *TreeClient) getObjectTaggingNode(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion) (*TreeNode, error) {
subtree, err := c.getSubTree(ctx, cnrID, versionTree, objVersion.ID, 1)
func (c *TreeClient) getTreeNode(ctx context.Context, cnrID *cid.ID, nodeID uint64, key string) (*TreeNode, error) {
subtree, err := c.getSubTree(ctx, cnrID, versionTree, nodeID, 1)
if err != nil {
return nil, err
}
var tagNode *TreeNode
var treeNode *TreeNode
for _, s := range subtree {
node, err := newTreeNode(s)
if err != nil {
return nil, err
}
if _, ok := node.Get(isTagKV); ok {
tagNode = node
if _, ok := node.Get(key); ok {
treeNode = node
break
}
}
return tagNode, nil
return treeNode, nil
}
func (c *TreeClient) GetVersions(ctx context.Context, cnrID *cid.ID, filepath string) ([]*data.NodeVersion, error) {
@ -666,17 +673,6 @@ func (c *TreeClient) GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID,
return result, nil
}
func (c *TreeClient) GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) {
meta := []string{oidKV}
path := pathFromName(objectName)
node, err := c.getLatestVersion(ctx, cnrID, systemTree, path, meta)
if err != nil {
return nil, err
}
return &node.BaseNodeVersion, nil
}
func (c *TreeClient) getLatestVersion(ctx context.Context, cnrID *cid.ID, treeID string, path, meta []string) (*data.NodeVersion, error) {
p := &getNodesParams{
CnrID: cnrID,
@ -723,22 +719,10 @@ func (c *TreeClient) AddVersion(ctx context.Context, cnrID *cid.ID, filepath str
return c.addVersion(ctx, cnrID, versionTree, filepath, version)
}
func (c *TreeClient) AddSystemVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *data.BaseNodeVersion) error {
newVersion := &data.NodeVersion{
BaseNodeVersion: *version,
IsUnversioned: true,
}
return c.addVersion(ctx, cnrID, systemTree, filepath, newVersion)
}
func (c *TreeClient) RemoveVersion(ctx context.Context, cnrID *cid.ID, id uint64) error {
return c.removeNode(ctx, cnrID, versionTree, id)
}
func (c *TreeClient) RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, id uint64) error {
return c.removeNode(ctx, cnrID, systemTree, id)
}
func (c *TreeClient) CreateMultipartUpload(ctx context.Context, cnrID *cid.ID, info *data.MultipartInfo) error {
path := pathFromName(info.Key)
meta := metaFromMultipart(info)
@ -869,6 +853,61 @@ func (c *TreeClient) DeleteMultipartUpload(ctx context.Context, cnrID *cid.ID, m
return c.removeNode(ctx, cnrID, systemTree, multipartNodeID)
}
func (c *TreeClient) PutLock(ctx context.Context, cnrID *cid.ID, nodeID uint64, lock *data.LockInfo) error {
meta := map[string]string{isLockKV: "true"}
if lock.LegalHoldOID != nil {
meta[legalHoldOIDKV] = lock.LegalHoldOID.EncodeToString()
} else if lock.RetentionOID != nil {
meta[retentionOIDKV] = lock.RetentionOID.EncodeToString()
meta[untilDateKV] = lock.UntilDate
if lock.IsCompliance {
meta[isComplianceKV] = "true"
}
}
if lock.ID == 0 {
_, err := c.addNode(ctx, cnrID, versionTree, nodeID, meta)
return err
}
return c.moveNode(ctx, cnrID, versionTree, lock.ID, nodeID, meta)
}
func (c *TreeClient) GetLock(ctx context.Context, cnrID *cid.ID, nodeID uint64) (*data.LockInfo, error) {
lockNode, err := c.getTreeNode(ctx, cnrID, nodeID, isLockKV)
if err != nil {
return nil, err
}
lockInfo := &data.LockInfo{}
if lockNode == nil {
return lockInfo, nil
}
lockInfo.ID = lockNode.ID
if legalHold, ok := lockNode.Get(legalHoldOIDKV); ok {
var legalHoldOID oid.ID
if err = legalHoldOID.DecodeString(legalHold); err != nil {
return nil, fmt.Errorf("invalid legal hold object id: %w", err)
}
lockInfo.LegalHoldOID = &legalHoldOID
}
if retention, ok := lockNode.Get(retentionOIDKV); ok {
var retentionOID oid.ID
if err = retentionOID.DecodeString(retention); err != nil {
return nil, fmt.Errorf("invalid retention object id: %w", err)
}
lockInfo.RetentionOID = &retentionOID
}
_, lockInfo.IsCompliance = lockNode.Get(isComplianceKV)
lockInfo.UntilDate, _ = lockNode.Get(untilDateKV)
return lockInfo, nil
}
func (c *TreeClient) Close() error {
if c.conn != nil {
return c.conn.Close()

58
internal/neofstest/tree/tree_mock.go
View file

@ -15,6 +15,7 @@ type TreeServiceMock struct {
settings map[string]*data.BucketSettings
versions map[string]map[string][]*data.NodeVersion
system map[string]map[string]*data.BaseNodeVersion
locks map[string]map[uint64]*data.LockInfo
}
func (t *TreeServiceMock) GetObjectTagging(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion) (map[string]string, error) {
@ -54,6 +55,7 @@ func NewTreeService() *TreeServiceMock {
settings: make(map[string]*data.BucketSettings),
versions: make(map[string]map[string][]*data.NodeVersion),
system: make(map[string]map[string]*data.BaseNodeVersion),
locks: make(map[string]map[uint64]*data.LockInfo),
}
}
@ -187,7 +189,6 @@ func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectNam
if !node.IsUnversioned {
result = append(result, node)
}
}
}
@ -200,38 +201,6 @@ func (t *TreeServiceMock) RemoveVersion(ctx context.Context, cnrID *cid.ID, node
panic("implement me")
}
func (t *TreeServiceMock) AddSystemVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *data.BaseNodeVersion) error {
cnrSystemMap, ok := t.system[cnrID.EncodeToString()]
if !ok {
t.system[cnrID.EncodeToString()] = map[string]*data.BaseNodeVersion{
objectName: newVersion,
}
return nil
}
cnrSystemMap[objectName] = newVersion
return nil
}
func (t *TreeServiceMock) GetSystemVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) {
cnrSystemMap, ok := t.system[cnrID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
sysVersion, ok := cnrSystemMap[objectName]
if !ok {
return nil, ErrNodeNotFound
}
return sysVersion, nil
}
func (t *TreeServiceMock) RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error {
panic("implement me")
}
func (t *TreeServiceMock) GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]*data.NodeVersion, error) {
panic("implement me")
}
@ -259,3 +228,26 @@ func (t *TreeServiceMock) GetParts(ctx context.Context, cnrID *cid.ID, multipart
func (t *TreeServiceMock) DeleteMultipartUpload(ctx context.Context, cnrID *cid.ID, multipartNodeID uint64) error {
panic("implement me")
}
func (t *TreeServiceMock) PutLock(ctx context.Context, cnrID *cid.ID, nodeID uint64, lock *data.LockInfo) error {
cnrLockMap, ok := t.locks[cnrID.EncodeToString()]
if !ok {
t.locks[cnrID.EncodeToString()] = map[uint64]*data.LockInfo{
nodeID: lock,
}
return nil
}
cnrLockMap[nodeID] = lock
return nil
}
func (t *TreeServiceMock) GetLock(ctx context.Context, cnrID *cid.ID, nodeID uint64) (*data.LockInfo, error) {
cnrLockMap, ok := t.locks[cnrID.EncodeToString()]
if !ok {
return nil, nil
}
return cnrLockMap[nodeID], nil
}