From dd534e8738652a7166b8a5a0eca97ae653918d20 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 26 May 2022 16:11:14 +0300 Subject: [PATCH] [#451] Handle lock objects using tree service Signed-off-by: Denis Kirillov --- api/cache/system.go | 20 +++ api/data/locking.go | 18 +- api/data/tree.go | 9 + api/handler/delete.go | 3 +- api/handler/get.go | 8 +- api/handler/head.go | 39 +++-- api/handler/locking.go | 162 +++++++---------- api/handler/locking_test.go | 23 ++- api/handler/multipart_upload.go | 8 +- api/handler/put.go | 16 +- api/handler/tagging.go | 25 +-- api/layer/cors.go | 18 +- api/layer/layer.go | 35 ++-- api/layer/locking_test.go | 26 +-- api/layer/notifications.go | 15 +- api/layer/object.go | 34 +--- api/layer/system_object.go | 250 ++++++++++++--------------- api/layer/tagging.go | 64 +++---- api/layer/tree_service.go | 5 +- api/layer/versioning.go | 59 +------ api/layer/versioning_test.go | 10 ++ internal/neofs/tree.go | 103 +++++++---- internal/neofstest/tree/tree_mock.go | 58 +++---- 23 files changed, 488 insertions(+), 520 deletions(-) diff --git a/api/cache/system.go b/api/cache/system.go index 0c86f231..95bd1396 100644 --- a/api/cache/system.go +++ b/api/cache/system.go @@ -56,6 +56,21 @@ func (o *SystemCache) GetObject(key string) *data.ObjectInfo { return result } +// GetLockInfo returns a cached object. +func (o *SystemCache) GetLockInfo(key string) *data.LockInfo { + entry, err := o.cache.Get(key) + if err != nil { + return nil + } + + result, ok := entry.(*data.LockInfo) + if !ok { + return nil + } + + return result +} + func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration { entry, err := o.cache.Get(key) if err != nil { @@ -124,6 +139,11 @@ func (o *SystemCache) PutObject(key string, obj *data.ObjectInfo) error { return o.cache.Set(key, obj) } +// PutLockInfo puts an object to cache. +func (o *SystemCache) PutLockInfo(key string, lockInfo *data.LockInfo) error { + return o.cache.Set(key, lockInfo) +} + func (o *SystemCache) PutCORS(key string, obj *data.CORSConfiguration) error { return o.cache.Set(key, obj) } diff --git a/api/data/locking.go b/api/data/locking.go index dbf8e568..992ded33 100644 --- a/api/data/locking.go +++ b/api/data/locking.go @@ -3,8 +3,6 @@ package data import ( "encoding/xml" "time" - - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) type ( @@ -36,9 +34,17 @@ type ( } ObjectLock struct { - Until time.Time - LegalHold bool - IsCompliance bool - Objects []oid.ID + LegalHold *LegalHoldLock + Retention *RetentionLock + } + + LegalHoldLock struct { + Enabled bool + } + + RetentionLock struct { + Until time.Time + IsCompliance bool + ByPassedGovernance bool } ) diff --git a/api/data/tree.go b/api/data/tree.go index 15630448..b0a7a553 100644 --- a/api/data/tree.go +++ b/api/data/tree.go @@ -62,3 +62,12 @@ type PartInfo struct { Number int OID oid.ID } + +// LockInfo is lock information to create appropriate tree node. +type LockInfo struct { + ID uint64 + LegalHoldOID *oid.ID + RetentionOID *oid.ID + UntilDate string + IsCompliance bool +} diff --git a/api/handler/delete.go b/api/handler/delete.go index b59ebbcf..36719cfe 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -4,6 +4,7 @@ import ( "encoding/xml" "net/http" "strconv" + "strings" "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/data" @@ -144,7 +145,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { func isErrObjectLocked(err error) bool { switch err.(type) { default: - return false + return strings.Contains(err.Error(), "object is locked") case apistatus.ObjectLocked, *apistatus.ObjectLocked: return true diff --git a/api/handler/get.go b/api/handler/get.go index 140a2f8d..a4cb8bd6 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -138,10 +138,10 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { return } - t := &data.ObjectTaggingInfo{ - CnrID: &info.CID, - ObjName: info.Name, - VersionID: info.Version(), + t := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: info.Name, + VersionID: info.Version(), } tagSet, err := h.obj.GetObjectTagging(r.Context(), t) diff --git a/api/handler/head.go b/api/handler/head.go index 762dead9..0f2cdc3e 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -62,10 +62,10 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { return } - t := &data.ObjectTaggingInfo{ - CnrID: &info.CID, - ObjName: info.Name, - VersionID: info.Version(), + t := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: info.Name, + VersionID: info.Version(), } tagSet, err := h.obj.GetObjectTagging(r.Context(), t) @@ -119,25 +119,28 @@ func (h *handler) setLockingHeaders(ctx context.Context, bktInfo *data.BucketInf } legalHold := &data.LegalHold{Status: legalHoldOff} - legalHoldInfo, err := h.obj.HeadSystemObject(ctx, bktInfo, objInfo.LegalHoldObject()) - if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) { - return err - } - - if legalHoldInfo != nil { - legalHold.Status = legalHoldOn - } - retention := &data.Retention{Mode: governanceMode} - retentionInfo, err := h.obj.HeadSystemObject(ctx, bktInfo, objInfo.RetentionObject()) + + p := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: objInfo.Name, + VersionID: objInfo.Version(), + } + + lockInfo, err := h.obj.GetLockInfo(ctx, p) if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) { return err } - if retentionInfo != nil { - retention.RetainUntilDate = retentionInfo.Headers[layer.AttributeRetainUntil] - if retentionInfo.Headers[layer.AttributeComplianceMode] != "" { - retention.Mode = complianceMode + if lockInfo != nil { + if lockInfo.LegalHoldOID != nil { + legalHold.Status = legalHoldOn + } + if lockInfo.RetentionOID != nil { + retention.RetainUntilDate = lockInfo.UntilDate + if lockInfo.IsCompliance { + retention.Mode = complianceMode + } } } diff --git a/api/handler/locking.go b/api/handler/locking.go index 18add91a..a7eba03a 100644 --- a/api/handler/locking.go +++ b/api/handler/locking.go @@ -11,7 +11,6 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/data" apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) const ( @@ -130,46 +129,21 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque return } - p := &layer.HeadObjectParams{ - BktInfo: bktInfo, - Object: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + p := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + } + lock := &data.ObjectLock{ + LegalHold: &data.LegalHoldLock{ + Enabled: legalHold.Status == legalHoldOn, + }, } - objInfo, err := h.obj.GetObjectInfo(r.Context(), p) - if err != nil { - h.logAndSendError(w, "could not get object info", reqInfo, err) + if err = h.obj.PutLockInfo(r.Context(), p, lock); err != nil { + h.logAndSendError(w, "couldn't head put legal hold", reqInfo, err) return } - - lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject()) - if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { - h.logAndSendError(w, "couldn't head lock object", reqInfo, err) - return - } - - if lockInfo == nil && legalHold.Status == legalHoldOff || - lockInfo != nil && legalHold.Status == legalHoldOn { - return - } - - if lockInfo != nil { - if err = h.obj.DeleteSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject()); err != nil { - h.logAndSendError(w, "couldn't delete legal hold", reqInfo, err) - return - } - } else { - ps := &layer.PutSystemObjectParams{ - BktInfo: bktInfo, - ObjName: objInfo.LegalHoldObject(), - Lock: &data.ObjectLock{LegalHold: true, Objects: []oid.ID{objInfo.ID}}, - Metadata: make(map[string]string), - } - if _, err = h.obj.PutSystemObject(r.Context(), ps); err != nil { - h.logAndSendError(w, "couldn't put legal hold", reqInfo, err) - return - } - } } func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) { @@ -187,26 +161,20 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque return } - p := &layer.HeadObjectParams{ - BktInfo: bktInfo, - Object: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + p := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), } - objInfo, err := h.obj.GetObjectInfo(r.Context(), p) + lockInfo, err := h.obj.GetLockInfo(r.Context(), p) if err != nil { - h.logAndSendError(w, "could not get object info", reqInfo, err) - return - } - - lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject()) - if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { h.logAndSendError(w, "couldn't head lock object", reqInfo, err) return } legalHold := &data.LegalHold{Status: legalHoldOff} - if lockInfo != nil { + if lockInfo.LegalHoldOID != nil { legalHold.Status = legalHoldOn } @@ -241,37 +209,13 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque return } - p := &layer.HeadObjectParams{ - BktInfo: bktInfo, - Object: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + p := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), } - objInfo, err := h.obj.GetObjectInfo(r.Context(), p) - if err != nil { - h.logAndSendError(w, "could not get object info", reqInfo, err) - return - } - lock.Objects = append(lock.Objects, objInfo.ID) - - lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.RetentionObject()) - if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { - h.logAndSendError(w, "couldn't head lock object", reqInfo, err) - return - } - - if err = checkLockInfo(lockInfo, r.Header); err != nil { - h.logAndSendError(w, "couldn't change lock mode", reqInfo, err) - return - } - - ps := &layer.PutSystemObjectParams{ - BktInfo: bktInfo, - ObjName: objInfo.RetentionObject(), - Lock: lock, - Metadata: make(map[string]string), - } - if _, err = h.obj.PutSystemObject(r.Context(), ps); err != nil { + if err = h.obj.PutLockInfo(r.Context(), p, lock); err != nil { h.logAndSendError(w, "couldn't put legal hold", reqInfo, err) return } @@ -308,29 +252,28 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque return } - p := &layer.HeadObjectParams{ - BktInfo: bktInfo, - Object: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + p := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), } - objInfo, err := h.obj.GetObjectInfo(r.Context(), p) - if err != nil { - h.logAndSendError(w, "could not get object info", reqInfo, err) - return - } - - lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.RetentionObject()) + lockInfo, err := h.obj.GetLockInfo(r.Context(), p) if err != nil { h.logAndSendError(w, "couldn't head lock object", reqInfo, err) return } + if lockInfo.RetentionOID == nil { + h.logAndSendError(w, "retention lock isn't set", reqInfo, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)) + return + } + retention := &data.Retention{ Mode: governanceMode, - RetainUntilDate: lockInfo.Headers[layer.AttributeRetainUntil], + RetainUntilDate: lockInfo.UntilDate, } - if lockInfo.Headers[layer.AttributeComplianceMode] != "" { + if lockInfo.IsCompliance { retention.Mode = complianceMode } @@ -379,21 +322,28 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf } if defaultConfig.Rule != nil && defaultConfig.Rule.DefaultRetention != nil { + retention := &data.RetentionLock{} defaultRetention := defaultConfig.Rule.DefaultRetention - objectLock.IsCompliance = defaultRetention.Mode == complianceMode + retention.IsCompliance = defaultRetention.Mode == complianceMode now := time.Now() if defaultRetention.Days != 0 { - objectLock.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration) + retention.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration) } else { - objectLock.Until = now.Add(time.Duration(defaultRetention.Years) * yearDuration) + retention.Until = now.Add(time.Duration(defaultRetention.Years) * yearDuration) } + objectLock.Retention = retention } - objectLock.LegalHold = header.Get(api.AmzObjectLockLegalHold) == legalHoldOn + if header.Get(api.AmzObjectLockLegalHold) == legalHoldOn { + objectLock.LegalHold = &data.LegalHoldLock{Enabled: true} + } mode := header.Get(api.AmzObjectLockMode) if mode != "" { - objectLock.IsCompliance = mode == complianceMode + if objectLock.Retention == nil { + objectLock.Retention = &data.RetentionLock{} + } + objectLock.Retention.IsCompliance = mode == complianceMode } until := header.Get(api.AmzObjectLockRetainUntilDate) @@ -402,7 +352,10 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf if err != nil { return nil, fmt.Errorf("invalid header %s: '%s'", api.AmzObjectLockRetainUntilDate, until) } - objectLock.Until = retentionDate + if objectLock.Retention == nil { + objectLock.Retention = &data.RetentionLock{} + } + objectLock.Retention.Until = retentionDate } return objectLock, nil @@ -424,9 +377,20 @@ func formObjectLockFromRetention(retention *data.Retention, header http.Header) return nil, fmt.Errorf("couldn't parse retain until date: %s", retention.RetainUntilDate) } + var bypass bool + if bypassStr := header.Get(api.AmzBypassGovernanceRetention); len(bypassStr) > 0 { + bypass, err = strconv.ParseBool(bypassStr) + if err != nil { + return nil, fmt.Errorf("couldn't parse bypass governance header: %w", err) + } + } + lock := &data.ObjectLock{ - Until: retentionDate, - IsCompliance: retention.Mode == complianceMode, + Retention: &data.RetentionLock{ + Until: retentionDate, + IsCompliance: retention.Mode == complianceMode, + ByPassedGovernance: bypass, + }, } return lock, nil diff --git a/api/handler/locking_test.go b/api/handler/locking_test.go index 5991036d..d5576a12 100644 --- a/api/handler/locking_test.go +++ b/api/handler/locking_test.go @@ -33,14 +33,17 @@ func TestFormObjectLock(t *testing.T) { bktInfo: &data.BucketInfo{ObjectLockEnabled: true}, config: &data.ObjectLockConfiguration{Rule: &data.ObjectLockRule{ DefaultRetention: &data.DefaultRetention{Mode: complianceMode, Days: 1}}}, - expectedLock: &data.ObjectLock{IsCompliance: true, Until: time.Now().Add(24 * time.Hour)}, + expectedLock: &data.ObjectLock{Retention: &data.RetentionLock{ + IsCompliance: true, + Until: time.Now().Add(24 * time.Hour)}}, }, { name: "default years", bktInfo: &data.BucketInfo{ObjectLockEnabled: true}, config: &data.ObjectLockConfiguration{Rule: &data.ObjectLockRule{ DefaultRetention: &data.DefaultRetention{Mode: governanceMode, Years: 1}}}, - expectedLock: &data.ObjectLock{Until: time.Now().Add(365 * 24 * time.Hour)}, + expectedLock: &data.ObjectLock{Retention: &data.RetentionLock{ + Until: time.Now().Add(365 * 24 * time.Hour)}}, }, { name: "basic override", @@ -51,7 +54,9 @@ func TestFormObjectLock(t *testing.T) { api.AmzObjectLockMode: {governanceMode}, api.AmzObjectLockLegalHold: {legalHoldOn}, }, - expectedLock: &data.ObjectLock{Until: time.Now(), LegalHold: true}, + expectedLock: &data.ObjectLock{ + LegalHold: &data.LegalHoldLock{Enabled: true}, + Retention: &data.RetentionLock{Until: time.Now()}}, }, { name: "lock disabled error", @@ -95,7 +100,9 @@ func TestFormObjectLockFromRetention(t *testing.T) { Mode: complianceMode, RetainUntilDate: time.Now().Format(time.RFC3339), }, - expectedLock: &data.ObjectLock{Until: time.Now(), IsCompliance: true}, + expectedLock: &data.ObjectLock{Retention: &data.RetentionLock{ + Until: time.Now(), + IsCompliance: true}}, }, { name: "basic governance", @@ -106,7 +113,7 @@ func TestFormObjectLockFromRetention(t *testing.T) { header: map[string][]string{ api.AmzBypassGovernanceRetention: {strconv.FormatBool(true)}, }, - expectedLock: &data.ObjectLock{Until: time.Now()}, + expectedLock: &data.ObjectLock{Retention: &data.RetentionLock{Until: time.Now()}}, }, { name: "error invalid mode", @@ -140,8 +147,10 @@ func TestFormObjectLockFromRetention(t *testing.T) { func assertObjectLocks(t *testing.T, expected, actual *data.ObjectLock) { require.Equal(t, expected.LegalHold, actual.LegalHold) - require.Equal(t, expected.IsCompliance, actual.IsCompliance) - require.InDelta(t, expected.Until.Unix(), actual.Until.Unix(), 1) + if expected.Retention != nil { + require.Equal(t, expected.Retention.IsCompliance, actual.Retention.IsCompliance) + require.InDelta(t, expected.Retention.Until.Unix(), actual.Retention.Until.Unix(), 1) + } } func TestCheckLockObject(t *testing.T) { diff --git a/api/handler/multipart_upload.go b/api/handler/multipart_upload.go index 107d7d32..ea7e29c3 100644 --- a/api/handler/multipart_upload.go +++ b/api/handler/multipart_upload.go @@ -370,10 +370,10 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http. } if len(uploadData.TagSet) != 0 { - t := &data.ObjectTaggingInfo{ - CnrID: &bktInfo.CID, - ObjName: objInfo.Name, - VersionID: objInfo.Version(), + t := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: objInfo.Name, + VersionID: objInfo.Version(), } if err = h.obj.PutObjectTagging(r.Context(), t, uploadData.TagSet); err != nil { h.logAndSendError(w, "could not put tagging file of completed multipart upload", reqInfo, err, additional...) diff --git a/api/handler/put.go b/api/handler/put.go index f8410d47..9b3fb46b 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -252,10 +252,10 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { } } - t := &data.ObjectTaggingInfo{ - CnrID: &info.CID, - ObjName: info.Name, - VersionID: info.Version(), + t := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: info.Name, + VersionID: info.Version(), } if tagSet != nil { if err = h.obj.PutObjectTagging(r.Context(), t, tagSet); err != nil { @@ -379,10 +379,10 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) { } } - t := &data.ObjectTaggingInfo{ - CnrID: &info.CID, - ObjName: info.Name, - VersionID: info.Version(), + t := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: info.Name, + VersionID: info.Version(), } if tagSet != nil { diff --git a/api/handler/tagging.go b/api/handler/tagging.go index 84edec73..cacc092a 100644 --- a/api/handler/tagging.go +++ b/api/handler/tagging.go @@ -11,6 +11,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" + "github.com/nspcc-dev/neofs-s3-gw/api/layer" "go.uber.org/zap" ) @@ -37,10 +38,10 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request return } - p := &data.ObjectTaggingInfo{ - CnrID: &bktInfo.CID, - ObjName: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get("versionId"), + p := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get("versionId"), } if err = h.obj.PutObjectTagging(r.Context(), p, tagSet); err != nil { @@ -73,10 +74,10 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request return } - p := &data.ObjectTaggingInfo{ - CnrID: &bktInfo.CID, - ObjName: reqInfo.ObjectName, - VersionID: versionID, + p := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: reqInfo.ObjectName, + VersionID: versionID, } tagSet, err := h.obj.GetObjectTagging(r.Context(), p) @@ -100,10 +101,10 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ return } - p := &data.ObjectTaggingInfo{ - CnrID: &bktInfo.CID, - ObjName: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get("versionId"), + p := &layer.ObjectVersion{ + BktInfo: bktInfo, + ObjectName: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get("versionId"), } if err = h.obj.DeleteObjectTagging(r.Context(), p); err != nil { diff --git a/api/layer/cors.go b/api/layer/cors.go index e2914887..1a8eb92c 100644 --- a/api/layer/cors.go +++ b/api/layer/cors.go @@ -36,21 +36,19 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error { return err } - s := &PutSystemObjectParams{ - BktInfo: p.BktInfo, - ObjName: p.BktInfo.CORSObjectName(), - Metadata: map[string]string{}, - Prefix: "", - Reader: &buf, - Size: int64(buf.Len()), + prm := PrmObjectCreate{ + Container: p.BktInfo.CID, + Creator: p.BktInfo.Owner, + Payload: p.Reader, + Filename: p.BktInfo.CORSObjectName(), } - obj, err := n.putSystemObjectIntoNeoFS(ctx, s) + objID, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo) if err != nil { return fmt.Errorf("put system object: %w", err) } - objIDToDelete, err := n.treeService.PutBucketCORS(ctx, &p.BktInfo.CID, &obj.ID) + objIDToDelete, err := n.treeService.PutBucketCORS(ctx, &p.BktInfo.CID, objID) if err != nil { return err } @@ -64,7 +62,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error { } } - if err := n.systemCache.PutCORS(systemObjectKey(p.BktInfo, s.ObjName), cors); err != nil { + if err := n.systemCache.PutCORS(systemObjectKey(p.BktInfo, prm.Filename), cors); err != nil { n.log.Error("couldn't cache system object", zap.Error(err)) } diff --git a/api/layer/layer.go b/api/layer/layer.go index 9adb7d08..9c7e67b7 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -90,6 +90,13 @@ type ( VersionID string } + // ObjectVersion stores object version info. + ObjectVersion struct { + BktInfo *data.BucketInfo + ObjectName string + VersionID string + } + // RangeParams stores range header request parameters. RangeParams struct { Start uint64 @@ -208,19 +215,20 @@ type ( DeleteBucket(ctx context.Context, p *DeleteBucketParams) error GetObject(ctx context.Context, p *GetObjectParams) error - HeadSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) (*data.ObjectInfo, error) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) + GetLockInfo(ctx context.Context, obj *ObjectVersion) (*data.LockInfo, error) + PutLockInfo(ctx context.Context, p *ObjectVersion, lock *data.ObjectLock) error + GetBucketTagging(ctx context.Context, cnrID *cid.ID) (map[string]string, error) PutBucketTagging(ctx context.Context, cnrID *cid.ID, tagSet map[string]string) error DeleteBucketTagging(ctx context.Context, cnrID *cid.ID) error - GetObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) (map[string]string, error) - PutObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo, tagSet map[string]string) error - DeleteObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) error + GetObjectTagging(ctx context.Context, p *ObjectVersion) (map[string]string, error) + PutObjectTagging(ctx context.Context, p *ObjectVersion, tagSet map[string]string) error + DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) - PutSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ObjectInfo, error) @@ -245,8 +253,7 @@ type ( ) const ( - tagPrefix = "S3-Tag-" - tagEmptyMark = "\\" + tagPrefix = "S3-Tag-" ) func (t *VersionedObject) String() string { @@ -520,17 +527,17 @@ func (n *layer) removeVersionIfFound(ctx context.Context, bkt *data.BucketInfo, deleteMarkVersion = obj.VersionID } - if err := n.treeService.RemoveVersion(ctx, &bkt.CID, version.ID); err != nil { - return deleteMarkVersion, err - } if err := n.objectDelete(ctx, bkt, version.OID); err != nil { return deleteMarkVersion, err } + if err := n.treeService.RemoveVersion(ctx, &bkt.CID, version.ID); err != nil { + return deleteMarkVersion, err + } - p := &data.ObjectTaggingInfo{ - CnrID: &bkt.CID, - ObjName: obj.Name, - VersionID: version.OID.EncodeToString(), + p := &ObjectVersion{ + BktInfo: bkt, + ObjectName: obj.Name, + VersionID: version.OID.EncodeToString(), } return deleteMarkVersion, n.DeleteObjectTagging(ctx, p) } diff --git a/api/layer/locking_test.go b/api/layer/locking_test.go index c6021b0c..edd0ab91 100644 --- a/api/layer/locking_test.go +++ b/api/layer/locking_test.go @@ -5,7 +5,6 @@ import ( "time" "github.com/nspcc-dev/neofs-s3-gw/api/data" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -19,18 +18,25 @@ func TestObjectLockAttributes(t *testing.T) { obj := tc.putObject([]byte("content obj1 v1")) - _, err = tc.layer.PutSystemObject(tc.ctx, &PutSystemObjectParams{ - BktInfo: tc.bktInfo, - ObjName: obj.RetentionObject(), - Metadata: make(map[string]string), - Lock: &data.ObjectLock{ - Until: time.Now(), - Objects: []oid.ID{obj.ID}, + p := &ObjectVersion{ + BktInfo: tc.bktInfo, + ObjectName: obj.Name, + VersionID: obj.Version(), + } + + lock := &data.ObjectLock{ + Retention: &data.RetentionLock{ + Until: time.Now(), }, - }) + } + + err = tc.layer.PutLockInfo(tc.ctx, p, lock) require.NoError(t, err) - lockObj := tc.getSystemObject(obj.RetentionObject()) + foundLock, err := tc.layer.GetLockInfo(tc.ctx, p) + require.NoError(t, err) + + lockObj := tc.getObjectByID(*foundLock.RetentionOID) require.NotNil(t, lockObj) expEpoch := false diff --git a/api/layer/notifications.go b/api/layer/notifications.go index 0747e7fe..f814eace 100644 --- a/api/layer/notifications.go +++ b/api/layer/notifications.go @@ -26,20 +26,19 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu sysName := p.BktInfo.NotificationConfigurationObjectName() - s := &PutSystemObjectParams{ - BktInfo: p.BktInfo, - ObjName: sysName, - Metadata: map[string]string{}, - Reader: bytes.NewReader(confXML), - Size: int64(len(confXML)), + prm := PrmObjectCreate{ + Container: p.BktInfo.CID, + Creator: p.BktInfo.Owner, + Payload: bytes.NewReader(confXML), + Filename: sysName, } - obj, err := n.putSystemObjectIntoNeoFS(ctx, s) + objID, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo) if err != nil { return err } - objIDToDelete, err := n.treeService.PutNotificationConfigurationNode(ctx, &p.BktInfo.CID, &obj.ID) + objIDToDelete, err := n.treeService.PutNotificationConfigurationNode(ctx, &p.BktInfo.CID, objID) if err != nil { return err } diff --git a/api/layer/object.go b/api/layer/object.go index 417e08d6..31f844bb 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -220,18 +220,15 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object zap.Error(err)) } - if p.Lock != nil { - objInfo := &data.ObjectInfo{ID: *id, Name: p.Object} - p.Lock.Objects = append(p.Lock.Objects, *id) - if p.Lock.LegalHold { - if err = n.putLockObject(ctx, p.BktInfo, objInfo.LegalHoldObject(), p.Lock); err != nil { - return nil, err - } + if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) { + objVersion := &ObjectVersion{ + BktInfo: p.BktInfo, + ObjectName: p.Object, + VersionID: id.EncodeToString(), } - if !p.Lock.Until.IsZero() { - if err = n.putLockObject(ctx, p.BktInfo, objInfo.RetentionObject(), p.Lock); err != nil { - return nil, err - } + + if err = n.PutLockInfo(ctx, objVersion, p.Lock); err != nil { + return nil, err } } @@ -266,21 +263,6 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object return objInfo, nil } -func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objName string, lock *data.ObjectLock) error { - ps := &PutSystemObjectParams{ - BktInfo: bktInfo, - ObjName: objName, - Lock: lock, - Metadata: make(map[string]string), - } - - if _, err := n.PutSystemObject(ctx, ps); err != nil { - return fmt.Errorf("coudln't add lock for '%s': %w", objName, err) - } - - return nil -} - func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ObjectInfo, error) { if addr := n.namesCache.Get(bkt.Name + "/" + objectName); addr != nil { if headInfo := n.objCache.Get(*addr); headInfo != nil { diff --git a/api/layer/system_object.go b/api/layer/system_object.go index b66ccaf7..af39d3eb 100644 --- a/api/layer/system_object.go +++ b/api/layer/system_object.go @@ -2,7 +2,6 @@ package layer import ( "context" - "encoding/hex" "encoding/xml" errorsStd "errors" "fmt" @@ -11,56 +10,123 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" - "github.com/nspcc-dev/neofs-s3-gw/internal/misc" - "github.com/nspcc-dev/neofs-sdk-go/object" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "go.uber.org/zap" ) const ( AttributeComplianceMode = ".s3-compliance-mode" - AttributeRetainUntil = ".s3-retain-until" AttributeExpirationEpoch = "__NEOFS__EXPIRATION_EPOCH" - AttributeSysTickEpoch = "__NEOFS__TICK_EPOCH" - AttributeSysTickTopic = "__NEOFS__TICK_TOPIC" ) -func (n *layer) PutSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) { - objInfo, err := n.putSystemObjectIntoNeoFS(ctx, p) +func (n *layer) PutLockInfo(ctx context.Context, objVersion *ObjectVersion, newLock *data.ObjectLock) error { + cnrID := objVersion.BktInfo.CID + versionNode, err := n.getNodeVersion(ctx, objVersion) if err != nil { - return nil, err + return err } - if err = n.systemCache.PutObject(systemObjectKey(p.BktInfo, p.ObjName), objInfo); err != nil { + lockInfo, err := n.treeService.GetLock(ctx, &cnrID, versionNode.ID) + if err != nil && !errorsStd.Is(err, ErrNodeNotFound) { + return err + } + + if lockInfo == nil { + lockInfo = &data.LockInfo{} + } + + if newLock.Retention != nil { + if lockInfo.RetentionOID != nil { + if lockInfo.IsCompliance { + return fmt.Errorf("you cannot change compliance mode") + } + if !newLock.Retention.ByPassedGovernance { + return fmt.Errorf("you cannot bypass governence mode") + } + + if len(lockInfo.UntilDate) > 0 { + parsedTime, err := time.Parse(time.RFC3339, lockInfo.UntilDate) + if err != nil { + return fmt.Errorf("couldn't parse time '%s': %w", lockInfo.UntilDate, err) + } + if parsedTime.After(newLock.Retention.Until) { + return fmt.Errorf("you couldn't short the until date") + } + } + } + lock := &data.ObjectLock{Retention: newLock.Retention} + if lockInfo.RetentionOID, err = n.putLockObject(ctx, objVersion.BktInfo, versionNode.OID, lock); err != nil { + return err + } + lockInfo.IsCompliance = newLock.Retention.IsCompliance + lockInfo.UntilDate = newLock.Retention.Until.UTC().Format(time.RFC3339) + } + + if newLock.LegalHold != nil { + if newLock.LegalHold.Enabled && lockInfo.LegalHoldOID == nil { + lock := &data.ObjectLock{LegalHold: newLock.LegalHold} + if lockInfo.LegalHoldOID, err = n.putLockObject(ctx, objVersion.BktInfo, versionNode.OID, lock); err != nil { + return err + } + } else if !newLock.LegalHold.Enabled && lockInfo.LegalHoldOID != nil { + if err = n.objectDelete(ctx, objVersion.BktInfo, *lockInfo.LegalHoldOID); err != nil { + return fmt.Errorf("couldn't delete lock object '%s' to remove legal hold: %w", lockInfo.LegalHoldOID.EncodeToString(), err) + } + lockInfo.LegalHoldOID = nil + } + } + + if err = n.treeService.PutLock(ctx, &cnrID, versionNode.ID, lockInfo); err != nil { + return fmt.Errorf("couldn't put lock into tree: %w", err) + } + + if err = n.systemCache.PutLockInfo(lockObjectKey(objVersion), lockInfo); err != nil { n.log.Error("couldn't cache system object", zap.Error(err)) } - return objInfo, nil + return nil } -func (n *layer) HeadSystemObject(ctx context.Context, bkt *data.BucketInfo, objName string) (*data.ObjectInfo, error) { - if objInfo := n.systemCache.GetObject(systemObjectKey(bkt, objName)); objInfo != nil { - return objInfo, nil +func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, lock *data.ObjectLock) (*oid.ID, error) { + prm := PrmObjectCreate{ + Container: bktInfo.CID, + Creator: bktInfo.Owner, + Locks: []oid.ID{objID}, } - node, err := n.treeService.GetSystemVersion(ctx, &bkt.CID, objName) - if err != nil { - if errorsStd.Is(err, ErrNodeNotFound) { - return nil, errors.GetAPIError(errors.ErrNoSuchKey) - } - return nil, err - } - - meta, err := n.objectHead(ctx, bkt, node.OID) + var err error + prm.Attributes, err = n.attributesFromLock(ctx, lock) if err != nil { return nil, err } - objInfo := objInfoFromMeta(bkt, meta) - if err = n.systemCache.PutObject(systemObjectKey(bkt, objName), objInfo); err != nil { + id, _, err := n.objectPutAndHash(ctx, prm, bktInfo) + return id, err +} + +func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) { + if lockInfo := n.systemCache.GetLockInfo(lockObjectKey(objVersion)); lockInfo != nil { + return lockInfo, nil + } + + versionNode, err := n.getNodeVersion(ctx, objVersion) + if err != nil { + return nil, err + } + + lockInfo, err := n.treeService.GetLock(ctx, &objVersion.BktInfo.CID, versionNode.ID) + if err != nil && !errorsStd.Is(err, ErrNodeNotFound) { + return nil, err + } + if lockInfo == nil { + lockInfo = &data.LockInfo{} + } + + if err = n.systemCache.PutLockInfo(lockObjectKey(objVersion), lockInfo); err != nil { n.log.Error("couldn't cache system object", zap.Error(err)) } - return objInfo, nil + return lockInfo, nil } func (n *layer) DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error { @@ -83,97 +149,6 @@ func (n *layer) DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo return nil } -func (n *layer) putSystemObjectIntoNeoFS(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) { - prm := PrmObjectCreate{ - Container: p.BktInfo.CID, - Creator: p.BktInfo.Owner, - Attributes: make([][2]string, 2, 2+len(p.Metadata)), - Payload: p.Reader, - } - - prm.Attributes[0][0], prm.Attributes[0][1] = objectSystemAttributeName, p.ObjName - prm.Attributes[1][0], prm.Attributes[1][1] = attrVersionsIgnore, "true" - - for k, v := range p.Metadata { - if !IsSystemHeader(k) { - k = p.Prefix + k - } - - if v == "" && p.Prefix == tagPrefix { - v = tagEmptyMark - } - - if p.Lock != nil && len(p.Lock.Objects) > 0 { - prm.Locks = p.Lock.Objects - - attrs, err := n.attributesFromLock(ctx, p.Lock) - if err != nil { - return nil, fmt.Errorf("get lock attributes: %w", err) - } - - prm.Attributes = append(prm.Attributes, attrs...) - } - - prm.Attributes = append(prm.Attributes, [2]string{k, v}) - } - - id, hash, err := n.objectPutAndHash(ctx, prm, p.BktInfo) - if err != nil { - return nil, err - } - - newVersion := &data.BaseNodeVersion{OID: *id} - if err = n.treeService.AddSystemVersion(ctx, &p.BktInfo.CID, p.ObjName, newVersion); err != nil { - return nil, fmt.Errorf("couldn't add new verion to tree service: %w", err) - } - - currentEpoch, _, err := n.neoFS.TimeToEpoch(ctx, time.Now().Add(time.Minute)) - if err != nil { - n.log.Warn("couldn't get creation epoch", - zap.String("bucket", p.BktInfo.Name), - zap.String("object", misc.SanitizeString(p.ObjName)), - zap.Error(err)) - } - - headers := make(map[string]string, len(p.Metadata)) - for _, attr := range prm.Attributes { - headers[attr[0]] = attr[1] - } - - return &data.ObjectInfo{ - ID: *id, - CID: p.BktInfo.CID, - - Owner: p.BktInfo.Owner, - Bucket: p.BktInfo.Name, - Name: p.ObjName, - Created: time.Now(), - CreationEpoch: currentEpoch, - Size: p.Size, - Headers: headers, - HashSum: hex.EncodeToString(hash), - }, nil -} - -func (n *layer) getSystemObjectFromNeoFS(ctx context.Context, bkt *data.BucketInfo, objName string) (*object.Object, error) { - versions, err := n.headSystemVersions(ctx, bkt, objName) - if err != nil { - return nil, err - } - - objInfo := versions.getLast() - - obj, err := n.objectGet(ctx, bkt, objInfo.ID) - if err != nil { - return nil, err - } - - if len(obj.Payload()) == 0 { - return nil, errors.GetAPIError(errors.ErrInternalError) - } - return obj, nil -} - func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo, sysName string) (*data.CORSConfiguration, error) { if cors := n.systemCache.GetCORS(systemObjectKey(bkt, sysName)); cors != nil { return cors, nil @@ -251,6 +226,11 @@ func systemObjectKey(bktInfo *data.BucketInfo, obj string) string { return bktInfo.Name + obj } +func lockObjectKey(objVersion *ObjectVersion) string { + // todo reconsider forming name since versionID can be "null" or "" + return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID +} + func (n *layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) { systemKey := systemObjectKey(bktInfo, bktInfo.SettingsObjectName()) if settings := n.systemCache.GetSettings(systemKey); settings != nil { @@ -289,26 +269,24 @@ func (n *layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) err } func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) ([][2]string, error) { - var result [][2]string - if !lock.Until.IsZero() { - _, exp, err := n.neoFS.TimeToEpoch(ctx, lock.Until) - if err != nil { - return nil, fmt.Errorf("fetch time to epoch: %w", err) - } - - attrs := [][2]string{ - {AttributeExpirationEpoch, strconv.FormatUint(exp, 10)}, - {AttributeRetainUntil, lock.Until.Format(time.RFC3339)}, - } - - result = append(result, attrs...) - if lock.IsCompliance { - attrCompliance := [2]string{ - AttributeComplianceMode, strconv.FormatBool(true), - } - result = append(result, attrCompliance) - } + if lock.Retention == nil { + return nil, nil } + _, exp, err := n.neoFS.TimeToEpoch(ctx, lock.Retention.Until) + if err != nil { + return nil, fmt.Errorf("fetch time to epoch: %w", err) + } + + result := [][2]string{ + {AttributeExpirationEpoch, strconv.FormatUint(exp, 10)}, + } + + if lock.Retention.IsCompliance { + attrCompliance := [2]string{ + AttributeComplianceMode, strconv.FormatBool(true), + } + result = append(result, attrCompliance) + } return result, nil } diff --git a/api/layer/tagging.go b/api/layer/tagging.go index 97301f2d..393e7a86 100644 --- a/api/layer/tagging.go +++ b/api/layer/tagging.go @@ -3,13 +3,14 @@ package layer import ( "context" errorsStd "errors" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "go.uber.org/zap" ) -func (n *layer) GetObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) (map[string]string, error) { +func (n *layer) GetObjectTagging(ctx context.Context, p *ObjectVersion) (map[string]string, error) { var ( err error tags map[string]string @@ -19,12 +20,12 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) return tags, nil } - version, err := n.getTaggedObjectVersion(ctx, p) + version, err := n.getNodeVersion(ctx, p) if err != nil { return nil, err } - tags, err = n.treeService.GetObjectTagging(ctx, p.CnrID, version) + tags, err = n.treeService.GetObjectTagging(ctx, &p.BktInfo.CID, version) if err != nil { if errorsStd.Is(err, ErrNodeNotFound) { return nil, errors.GetAPIError(errors.ErrNoSuchKey) @@ -39,13 +40,13 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) return tags, nil } -func (n *layer) PutObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo, tagSet map[string]string) error { - version, err := n.getTaggedObjectVersion(ctx, p) +func (n *layer) PutObjectTagging(ctx context.Context, p *ObjectVersion, tagSet map[string]string) error { + version, err := n.getNodeVersion(ctx, p) if err != nil { return err } - err = n.treeService.PutObjectTagging(ctx, p.CnrID, version, tagSet) + err = n.treeService.PutObjectTagging(ctx, &p.BktInfo.CID, version, tagSet) if err != nil { if errorsStd.Is(err, ErrNodeNotFound) { return errors.GetAPIError(errors.ErrNoSuchKey) @@ -60,13 +61,13 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo, return nil } -func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectTaggingInfo) error { - version, err := n.getTaggedObjectVersion(ctx, p) +func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error { + version, err := n.getNodeVersion(ctx, p) if err != nil { return err } - err = n.treeService.DeleteObjectTagging(ctx, p.CnrID, version) + err = n.treeService.DeleteObjectTagging(ctx, &p.BktInfo.CID, version) if err != nil { if errorsStd.Is(err, ErrNodeNotFound) { return errors.GetAPIError(errors.ErrNoSuchKey) @@ -118,50 +119,41 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, cnrID *cid.ID) error { return n.treeService.DeleteBucketTagging(ctx, cnrID) } -func objectTaggingCacheKey(p *data.ObjectTaggingInfo) string { - return ".tagset." + p.CnrID.EncodeToString() + "." + p.ObjName + "." + p.VersionID +func objectTaggingCacheKey(p *ObjectVersion) string { + return ".tagset." + p.BktInfo.CID.EncodeToString() + "." + p.ObjectName + "." + p.VersionID } func bucketTaggingCacheKey(cnrID *cid.ID) string { return ".tagset." + cnrID.EncodeToString() } -func (n *layer) getTaggedObjectVersion(ctx context.Context, p *data.ObjectTaggingInfo) (*data.NodeVersion, error) { - var ( - err error - version *data.NodeVersion - ) +func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (*data.NodeVersion, error) { + var err error + var version *data.NodeVersion - if p.VersionID == unversionedObjectVersionID { - if version, err = n.treeService.GetUnversioned(ctx, p.CnrID, p.ObjName); err != nil { - if errorsStd.Is(err, ErrNodeNotFound) { - return nil, errors.GetAPIError(errors.ErrNoSuchKey) - } - return nil, err - } - } else if len(p.VersionID) == 0 { - if version, err = n.treeService.GetLatestVersion(ctx, p.CnrID, p.ObjName); err != nil { - if errorsStd.Is(err, ErrNodeNotFound) { - return nil, errors.GetAPIError(errors.ErrNoSuchKey) - } - return nil, err - } + if objVersion.VersionID == unversionedObjectVersionID { + version, err = n.treeService.GetUnversioned(ctx, &objVersion.BktInfo.CID, objVersion.ObjectName) + } else if len(objVersion.VersionID) == 0 { + version, err = n.treeService.GetLatestVersion(ctx, &objVersion.BktInfo.CID, objVersion.ObjectName) } else { - versions, err := n.treeService.GetVersions(ctx, p.CnrID, p.ObjName) - if err != nil { - return nil, err + versions, err2 := n.treeService.GetVersions(ctx, &objVersion.BktInfo.CID, objVersion.ObjectName) + if err2 != nil { + return nil, err2 } for _, v := range versions { - if v.OID.EncodeToString() == p.VersionID { + if v.OID.EncodeToString() == objVersion.VersionID { version = v break } } + if version == nil { + err = errors.GetAPIError(errors.ErrNoSuchKey) + } } - if version == nil || version.DeleteMarker != nil { + if err == nil && version.DeleteMarker != nil || errorsStd.Is(err, ErrNodeNotFound) { return nil, errors.GetAPIError(errors.ErrNoSuchKey) } - return version, nil + return version, err } diff --git a/api/layer/tree_service.go b/api/layer/tree_service.go index b98c1e0a..c328e680 100644 --- a/api/layer/tree_service.go +++ b/api/layer/tree_service.go @@ -46,9 +46,8 @@ type TreeService interface { AddVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *data.NodeVersion) error RemoveVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error - AddSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *data.BaseNodeVersion) error - GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) - RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error + PutLock(ctx context.Context, cnrID *cid.ID, nodeID uint64, lock *data.LockInfo) error + GetLock(ctx context.Context, cnrID *cid.ID, nodeID uint64) (*data.LockInfo, error) CreateMultipartUpload(ctx context.Context, cnrID *cid.ID, info *data.MultipartInfo) error DeleteMultipartUpload(ctx context.Context, cnrID *cid.ID, multipartNodeID uint64) error diff --git a/api/layer/versioning.go b/api/layer/versioning.go index 4ffca683..fe887748 100644 --- a/api/layer/versioning.go +++ b/api/layer/versioning.go @@ -4,7 +4,6 @@ import ( "context" "math" "sort" - "strconv" "strings" "github.com/nspcc-dev/neofs-s3-gw/api/data" @@ -42,13 +41,12 @@ const ( VersionsDeleteMarkAttr = "S3-Versions-delete-mark" DelMarkFullObject = "*" - unversionedObjectVersionID = "null" - objectSystemAttributeName = "S3-System-name" - attrVersionsIgnore = "S3-Versions-ignore" - attrSettingsVersioningEnabled = "S3-Settings-Versioning-enabled" - versionsDelAttr = "S3-Versions-del" - versionsAddAttr = "S3-Versions-add" - versionsUnversionedAttr = "S3-Versions-unversioned" + unversionedObjectVersionID = "null" + objectSystemAttributeName = "S3-System-name" + attrVersionsIgnore = "S3-Versions-ignore" + versionsDelAttr = "S3-Versions-del" + versionsAddAttr = "S3-Versions-add" + versionsUnversionedAttr = "S3-Versions-unversioned" ) func newObjectVersions(name string) *objectVersions { @@ -213,56 +211,11 @@ func (v *objectVersions) existedVersions() []string { return res } -func (v *objectVersions) getFiltered(reverse bool) []*data.ObjectInfo { - if len(v.objects) == 0 { - return nil - } - - v.sort() - existedVersions := v.existedVersions() - res := make([]*data.ObjectInfo, 0, len(v.objects)) - - for _, version := range v.objects { - delMark := version.Headers[VersionsDeleteMarkAttr] - if contains(existedVersions, version.Version()) && (delMark == DelMarkFullObject || delMark == "") { - res = append(res, version) - } - } - - if reverse { - for i, j := 0, len(res)-1; i < j; i, j = i+1, j-1 { - res[i], res[j] = res[j], res[i] - } - } - - return res -} - func (v *objectVersions) getAddHeader() string { v.sort() return strings.Join(v.addList, ",") } -func (n *layer) PutBucketVersioning(ctx context.Context, p *PutSettingsParams) (*data.ObjectInfo, error) { - metadata := map[string]string{ - attrSettingsVersioningEnabled: strconv.FormatBool(p.Settings.VersioningEnabled), - } - - s := &PutSystemObjectParams{ - BktInfo: p.BktInfo, - ObjName: p.BktInfo.SettingsObjectName(), - Metadata: metadata, - Prefix: "", - Reader: nil, - } - - return n.PutSystemObject(ctx, s) -} - -func (n *layer) GetBucketVersioning(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) { - return n.GetBucketSettings(ctx, bktInfo) -} - func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { var ( allObjects = make([]*data.ObjectInfo, 0, p.MaxKeys) diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go index 5dc7f50e..e24c0c65 100644 --- a/api/layer/versioning_test.go +++ b/api/layer/versioning_test.go @@ -127,6 +127,16 @@ func (tc *testContext) getSystemObject(objectName string) *object.Object { return nil } +func (tc *testContext) getObjectByID(objID oid.ID) *object.Object { + for _, obj := range tc.testNeoFS.Objects() { + id, _ := obj.ID() + if id.Equals(objID) { + return obj + } + } + return nil +} + type testContext struct { t *testing.T ctx context.Context diff --git a/internal/neofs/tree.go b/internal/neofs/tree.go index 5a57786c..26fdcaf8 100644 --- a/internal/neofs/tree.go +++ b/internal/neofs/tree.go @@ -56,6 +56,13 @@ const ( uploadIDKV = "UploadId" partNumberKV = "Number" + // keys for lock. + isLockKV = "IsLock" + legalHoldOIDKV = "LegalHoldOID" + retentionOIDKV = "RetentionOID" + untilDateKV = "UntilDate" + isComplianceKV = "IsCompliance" + // keys for delete marker nodes. isDeleteMarkerKV = "IdDeleteMarker" filePathKV = "FilePath" @@ -339,7 +346,7 @@ func (c *TreeClient) DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid. } func (c *TreeClient) GetObjectTagging(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion) (map[string]string, error) { - tagNode, err := c.getObjectTaggingNode(ctx, cnrID, objVersion) + tagNode, err := c.getTreeNode(ctx, cnrID, objVersion.ID, isTagKV) if err != nil { return nil, err } @@ -360,7 +367,7 @@ func (c *TreeClient) GetObjectTagging(ctx context.Context, cnrID *cid.ID, objVer } func (c *TreeClient) PutObjectTagging(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion, tagSet map[string]string) error { - tagNode, err := c.getObjectTaggingNode(ctx, cnrID, objVersion) + tagNode, err := c.getTreeNode(ctx, cnrID, objVersion.ID, isTagKV) if err != nil { return err } @@ -382,7 +389,7 @@ func (c *TreeClient) PutObjectTagging(ctx context.Context, cnrID *cid.ID, objVer } func (c *TreeClient) DeleteObjectTagging(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion) error { - tagNode, err := c.getObjectTaggingNode(ctx, cnrID, objVersion) + tagNode, err := c.getTreeNode(ctx, cnrID, objVersion.ID, isTagKV) if err != nil { return err } @@ -450,26 +457,26 @@ func (c *TreeClient) DeleteBucketTagging(ctx context.Context, cnrID *cid.ID) err return nil } -func (c *TreeClient) getObjectTaggingNode(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion) (*TreeNode, error) { - subtree, err := c.getSubTree(ctx, cnrID, versionTree, objVersion.ID, 1) +func (c *TreeClient) getTreeNode(ctx context.Context, cnrID *cid.ID, nodeID uint64, key string) (*TreeNode, error) { + subtree, err := c.getSubTree(ctx, cnrID, versionTree, nodeID, 1) if err != nil { return nil, err } - var tagNode *TreeNode + var treeNode *TreeNode for _, s := range subtree { node, err := newTreeNode(s) if err != nil { return nil, err } - if _, ok := node.Get(isTagKV); ok { - tagNode = node + if _, ok := node.Get(key); ok { + treeNode = node break } } - return tagNode, nil + return treeNode, nil } func (c *TreeClient) GetVersions(ctx context.Context, cnrID *cid.ID, filepath string) ([]*data.NodeVersion, error) { @@ -666,17 +673,6 @@ func (c *TreeClient) GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID, return result, nil } -func (c *TreeClient) GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) { - meta := []string{oidKV} - path := pathFromName(objectName) - - node, err := c.getLatestVersion(ctx, cnrID, systemTree, path, meta) - if err != nil { - return nil, err - } - return &node.BaseNodeVersion, nil -} - func (c *TreeClient) getLatestVersion(ctx context.Context, cnrID *cid.ID, treeID string, path, meta []string) (*data.NodeVersion, error) { p := &getNodesParams{ CnrID: cnrID, @@ -723,22 +719,10 @@ func (c *TreeClient) AddVersion(ctx context.Context, cnrID *cid.ID, filepath str return c.addVersion(ctx, cnrID, versionTree, filepath, version) } -func (c *TreeClient) AddSystemVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *data.BaseNodeVersion) error { - newVersion := &data.NodeVersion{ - BaseNodeVersion: *version, - IsUnversioned: true, - } - return c.addVersion(ctx, cnrID, systemTree, filepath, newVersion) -} - func (c *TreeClient) RemoveVersion(ctx context.Context, cnrID *cid.ID, id uint64) error { return c.removeNode(ctx, cnrID, versionTree, id) } -func (c *TreeClient) RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, id uint64) error { - return c.removeNode(ctx, cnrID, systemTree, id) -} - func (c *TreeClient) CreateMultipartUpload(ctx context.Context, cnrID *cid.ID, info *data.MultipartInfo) error { path := pathFromName(info.Key) meta := metaFromMultipart(info) @@ -869,6 +853,61 @@ func (c *TreeClient) DeleteMultipartUpload(ctx context.Context, cnrID *cid.ID, m return c.removeNode(ctx, cnrID, systemTree, multipartNodeID) } +func (c *TreeClient) PutLock(ctx context.Context, cnrID *cid.ID, nodeID uint64, lock *data.LockInfo) error { + meta := map[string]string{isLockKV: "true"} + + if lock.LegalHoldOID != nil { + meta[legalHoldOIDKV] = lock.LegalHoldOID.EncodeToString() + } else if lock.RetentionOID != nil { + meta[retentionOIDKV] = lock.RetentionOID.EncodeToString() + meta[untilDateKV] = lock.UntilDate + if lock.IsCompliance { + meta[isComplianceKV] = "true" + } + } + + if lock.ID == 0 { + _, err := c.addNode(ctx, cnrID, versionTree, nodeID, meta) + return err + } + + return c.moveNode(ctx, cnrID, versionTree, lock.ID, nodeID, meta) +} + +func (c *TreeClient) GetLock(ctx context.Context, cnrID *cid.ID, nodeID uint64) (*data.LockInfo, error) { + lockNode, err := c.getTreeNode(ctx, cnrID, nodeID, isLockKV) + if err != nil { + return nil, err + } + + lockInfo := &data.LockInfo{} + if lockNode == nil { + return lockInfo, nil + } + lockInfo.ID = lockNode.ID + + if legalHold, ok := lockNode.Get(legalHoldOIDKV); ok { + var legalHoldOID oid.ID + if err = legalHoldOID.DecodeString(legalHold); err != nil { + return nil, fmt.Errorf("invalid legal hold object id: %w", err) + } + lockInfo.LegalHoldOID = &legalHoldOID + } + + if retention, ok := lockNode.Get(retentionOIDKV); ok { + var retentionOID oid.ID + if err = retentionOID.DecodeString(retention); err != nil { + return nil, fmt.Errorf("invalid retention object id: %w", err) + } + lockInfo.RetentionOID = &retentionOID + } + + _, lockInfo.IsCompliance = lockNode.Get(isComplianceKV) + lockInfo.UntilDate, _ = lockNode.Get(untilDateKV) + + return lockInfo, nil +} + func (c *TreeClient) Close() error { if c.conn != nil { return c.conn.Close() diff --git a/internal/neofstest/tree/tree_mock.go b/internal/neofstest/tree/tree_mock.go index e4870224..89b324eb 100644 --- a/internal/neofstest/tree/tree_mock.go +++ b/internal/neofstest/tree/tree_mock.go @@ -15,6 +15,7 @@ type TreeServiceMock struct { settings map[string]*data.BucketSettings versions map[string]map[string][]*data.NodeVersion system map[string]map[string]*data.BaseNodeVersion + locks map[string]map[uint64]*data.LockInfo } func (t *TreeServiceMock) GetObjectTagging(ctx context.Context, cnrID *cid.ID, objVersion *data.NodeVersion) (map[string]string, error) { @@ -54,6 +55,7 @@ func NewTreeService() *TreeServiceMock { settings: make(map[string]*data.BucketSettings), versions: make(map[string]map[string][]*data.NodeVersion), system: make(map[string]map[string]*data.BaseNodeVersion), + locks: make(map[string]map[uint64]*data.LockInfo), } } @@ -187,7 +189,6 @@ func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectNam if !node.IsUnversioned { result = append(result, node) } - } } @@ -200,38 +201,6 @@ func (t *TreeServiceMock) RemoveVersion(ctx context.Context, cnrID *cid.ID, node panic("implement me") } -func (t *TreeServiceMock) AddSystemVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *data.BaseNodeVersion) error { - cnrSystemMap, ok := t.system[cnrID.EncodeToString()] - if !ok { - t.system[cnrID.EncodeToString()] = map[string]*data.BaseNodeVersion{ - objectName: newVersion, - } - return nil - } - - cnrSystemMap[objectName] = newVersion - - return nil -} - -func (t *TreeServiceMock) GetSystemVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) { - cnrSystemMap, ok := t.system[cnrID.EncodeToString()] - if !ok { - return nil, ErrNodeNotFound - } - - sysVersion, ok := cnrSystemMap[objectName] - if !ok { - return nil, ErrNodeNotFound - } - - return sysVersion, nil -} - -func (t *TreeServiceMock) RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error { - panic("implement me") -} - func (t *TreeServiceMock) GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]*data.NodeVersion, error) { panic("implement me") } @@ -259,3 +228,26 @@ func (t *TreeServiceMock) GetParts(ctx context.Context, cnrID *cid.ID, multipart func (t *TreeServiceMock) DeleteMultipartUpload(ctx context.Context, cnrID *cid.ID, multipartNodeID uint64) error { panic("implement me") } + +func (t *TreeServiceMock) PutLock(ctx context.Context, cnrID *cid.ID, nodeID uint64, lock *data.LockInfo) error { + cnrLockMap, ok := t.locks[cnrID.EncodeToString()] + if !ok { + t.locks[cnrID.EncodeToString()] = map[uint64]*data.LockInfo{ + nodeID: lock, + } + return nil + } + + cnrLockMap[nodeID] = lock + + return nil +} + +func (t *TreeServiceMock) GetLock(ctx context.Context, cnrID *cid.ID, nodeID uint64) (*data.LockInfo, error) { + cnrLockMap, ok := t.locks[cnrID.EncodeToString()] + if !ok { + return nil, nil + } + + return cnrLockMap[nodeID], nil +}