From 7d6271be8a35fbeb42b5dc9ca3d995b9b69093ba Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 28 Feb 2022 16:39:04 +0300 Subject: [PATCH] [#195] Implement PUT, GET locks to certain object Signed-off-by: Denis Kirillov --- api/data/info.go | 6 + api/data/locking.go | 17 ++- api/handler/locking.go | 267 +++++++++++++++++++++++++++++++++++ api/handler/unimplemented.go | 16 --- api/headers.go | 1 + api/layer/cors.go | 2 +- api/layer/layer.go | 26 ++-- api/layer/object.go | 1 + api/layer/system_object.go | 35 ++++- api/layer/versioning.go | 2 +- 10 files changed, 337 insertions(+), 36 deletions(-) diff --git a/api/data/info.go b/api/data/info.go index f8ede5a7..d12395fd 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -104,3 +104,9 @@ func (o *ObjectInfo) Address() *address.Address { // TagsObject returns name of system object for tags. func (o *ObjectInfo) TagsObject() string { return ".tagset." + o.Name + "." + o.Version() } + +// LegalHoldObject returns name of system object for lock object. +func (o *ObjectInfo) LegalHoldObject() string { return ".lock." + o.Name + "." + o.Version() } + +// RetentionObject returns name of system object for retention lock object. +func (o *ObjectInfo) RetentionObject() string { return ".retention." + o.Name + "." + o.Version() } diff --git a/api/data/locking.go b/api/data/locking.go index 40686399..74422cad 100644 --- a/api/data/locking.go +++ b/api/data/locking.go @@ -1,9 +1,13 @@ package data -import "time" +import ( + "encoding/xml" + "time" +) type ( ObjectLockConfiguration struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ObjectLockConfiguration" json:"-"` ObjectLockEnabled string `xml:"ObjectLockEnabled" json:"ObjectLockEnabled"` Rule *ObjectLockRule `xml:"Rule" json:"Rule"` } @@ -18,6 +22,17 @@ type ( Years int64 `xml:"Years" json:"Years"` } + LegalHold struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LegalHold" json:"-"` + Status string `xml:"Status" json:"Status"` + } + + Retention struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Retention" json:"-"` + Mode string `xml:"Mode" json:"Mode"` + RetainUntilDate string `xml:"RetainUntilDate" json:"RetainUntilDate"` + } + ObjectLock struct { Until time.Time LegalHold bool diff --git a/api/handler/locking.go b/api/handler/locking.go index d1bcd299..3c903aa5 100644 --- a/api/handler/locking.go +++ b/api/handler/locking.go @@ -4,6 +4,7 @@ import ( "encoding/xml" "fmt" "net/http" + "strconv" "time" "github.com/nspcc-dev/neofs-s3-gw/api" @@ -20,6 +21,7 @@ const ( governanceMode = "GOVERNANCE" complianceMode = "COMPLIANCE" legalHoldOn = "ON" + legalHoldOff = "OFF" ) func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) { @@ -108,6 +110,240 @@ func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt } } +func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + + bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil { + h.logAndSendError(w, "expected owner doesn't match", reqInfo, err) + return + } + + if !bktInfo.ObjectLockEnabled { + h.logAndSendError(w, "object lock disabled", reqInfo, + apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)) + return + } + + legalHold := &data.LegalHold{} + if err = xml.NewDecoder(r.Body).Decode(legalHold); err != nil { + h.logAndSendError(w, "couldn't parse legal hold configuration", reqInfo, err) + return + } + + if legalHold.Status != legalHoldOn && legalHold.Status != legalHoldOff { + h.logAndSendError(w, "invalid legal hold status", reqInfo, + fmt.Errorf("invalid status %s", legalHold.Status)) + return + } + + p := &layer.HeadObjectParams{ + Bucket: reqInfo.BucketName, + Object: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + } + + objInfo, err := h.obj.GetObjectInfo(r.Context(), p) + if err != nil { + h.logAndSendError(w, "could not get object info", reqInfo, err) + return + } + + lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject()) + if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { + h.logAndSendError(w, "couldn't head lock object", reqInfo, err) + return + } + + if lockInfo == nil && legalHold.Status == legalHoldOff || + lockInfo != nil && legalHold.Status == legalHoldOn { + return + } + + if lockInfo != nil { + if err = h.obj.DeleteSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject()); err != nil { + h.logAndSendError(w, "couldn't delete legal hold", reqInfo, err) + return + } + } else { + ps := &layer.PutSystemObjectParams{ + BktInfo: bktInfo, + ObjName: objInfo.LegalHoldObject(), + Lock: &data.ObjectLock{LegalHold: true}, + } + if _, err = h.obj.PutSystemObject(r.Context(), ps); err != nil { + h.logAndSendError(w, "couldn't put legal hold", reqInfo, err) + return + } + } +} + +func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + + bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil { + h.logAndSendError(w, "expected owner doesn't match", reqInfo, err) + return + } + + if !bktInfo.ObjectLockEnabled { + h.logAndSendError(w, "object lock disabled", reqInfo, + apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)) + return + } + + p := &layer.HeadObjectParams{ + Bucket: reqInfo.BucketName, + Object: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + } + + objInfo, err := h.obj.GetObjectInfo(r.Context(), p) + if err != nil { + h.logAndSendError(w, "could not get object info", reqInfo, err) + return + } + + lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject()) + if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { + h.logAndSendError(w, "couldn't head lock object", reqInfo, err) + return + } + + legalHold := &data.LegalHold{Status: legalHoldOff} + if lockInfo != nil { + legalHold.Status = legalHoldOn + } + + if err = api.EncodeToResponse(w, legalHold); err != nil { + h.logAndSendError(w, "something went wrong", reqInfo, err) + } +} + +func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + + bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil { + h.logAndSendError(w, "expected owner doesn't match", reqInfo, err) + return + } + + if !bktInfo.ObjectLockEnabled { + h.logAndSendError(w, "object lock disabled", reqInfo, + apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)) + return + } + + retention := &data.Retention{} + if err = xml.NewDecoder(r.Body).Decode(retention); err != nil { + h.logAndSendError(w, "couldn't parse object retention", reqInfo, err) + return + } + + lock, err := formObjectLockFromRetention(retention, r.Header) + if err != nil { + h.logAndSendError(w, "invalid retention configuration", reqInfo, err) + return + } + + p := &layer.HeadObjectParams{ + Bucket: reqInfo.BucketName, + Object: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + } + + objInfo, err := h.obj.GetObjectInfo(r.Context(), p) + if err != nil { + h.logAndSendError(w, "could not get object info", reqInfo, err) + return + } + + lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.RetentionObject()) + if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { + h.logAndSendError(w, "couldn't head lock object", reqInfo, err) + return + } + + if lockInfo != nil && lockInfo.Headers[layer.AttributeComplianceMode] != "" { + h.logAndSendError(w, "couldn't change compliance lock mode", reqInfo, err) + return + } + + ps := &layer.PutSystemObjectParams{ + BktInfo: bktInfo, + ObjName: objInfo.RetentionObject(), + Lock: lock, + } + if _, err = h.obj.PutSystemObject(r.Context(), ps); err != nil { + h.logAndSendError(w, "couldn't put legal hold", reqInfo, err) + return + } +} + +func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + + bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil { + h.logAndSendError(w, "expected owner doesn't match", reqInfo, err) + return + } + + if !bktInfo.ObjectLockEnabled { + h.logAndSendError(w, "object lock disabled", reqInfo, + apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)) + return + } + + p := &layer.HeadObjectParams{ + Bucket: reqInfo.BucketName, + Object: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + } + + objInfo, err := h.obj.GetObjectInfo(r.Context(), p) + if err != nil { + h.logAndSendError(w, "could not get object info", reqInfo, err) + return + } + + lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.RetentionObject()) + if err != nil { + h.logAndSendError(w, "couldn't head lock object", reqInfo, err) + return + } + + retention := &data.Retention{ + Mode: governanceMode, + RetainUntilDate: lockInfo.Headers[layer.AttributeRetainUntil], + } + if lockInfo.Headers[layer.AttributeComplianceMode] != "" { + retention.Mode = complianceMode + } + + if err = api.EncodeToResponse(w, retention); err != nil { + h.logAndSendError(w, "something went wrong", reqInfo, err) + } +} + func checkLockConfiguration(conf *data.ObjectLockConfiguration) error { if conf.ObjectLockEnabled != "" && conf.ObjectLockEnabled != enabledValue { return fmt.Errorf("invalid ObjectLockEnabled value: %s", conf.ObjectLockEnabled) @@ -180,3 +416,34 @@ func existLockHeaders(header http.Header) bool { header.Get(api.AmzObjectLockLegalHold) != "" || header.Get(api.AmzObjectLockRetainUntilDate) != "" } + +func formObjectLockFromRetention(retention *data.Retention, header http.Header) (*data.ObjectLock, error) { + var err error + var bypassGovernance bool + bypass := header.Get(api.AmzBypassGovernanceRetention) + if bypass != "" { + if bypassGovernance, err = strconv.ParseBool(bypass); err != nil { + return nil, fmt.Errorf("couldn't parse '%s' header", api.AmzBypassGovernanceRetention) + } + } + + if retention.Mode != governanceMode && retention.Mode != complianceMode { + return nil, fmt.Errorf("invalid retention mode: %s", retention.Mode) + } + + retentionDate, err := time.Parse(time.RFC3339, retention.RetainUntilDate) + if err != nil { + return nil, fmt.Errorf("couldn't parse retain until date: %s", retention.RetainUntilDate) + } + + lock := &data.ObjectLock{ + Until: retentionDate, + IsCompliance: retention.Mode == complianceMode, + } + + if !lock.IsCompliance && !bypassGovernance { + return nil, fmt.Errorf("you cannot bypase governance mode") + } + + return lock, nil +} diff --git a/api/handler/unimplemented.go b/api/handler/unimplemented.go index b849e53a..6bbc9c0d 100644 --- a/api/handler/unimplemented.go +++ b/api/handler/unimplemented.go @@ -11,22 +11,6 @@ func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Requ h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } -func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - -func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - -func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - -func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } diff --git a/api/headers.go b/api/headers.go index e946e5ac..ddee405d 100644 --- a/api/headers.go +++ b/api/headers.go @@ -50,6 +50,7 @@ const ( AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold" AmzObjectLockMode = "X-Amz-Object-Lock-Mode" AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date" + AmzBypassGovernanceRetention = "X-Amz-Bypass-Governance-Retention" ContainerID = "X-Container-Id" diff --git a/api/layer/cors.go b/api/layer/cors.go index 7a5b757c..1c21caf3 100644 --- a/api/layer/cors.go +++ b/api/layer/cors.go @@ -72,7 +72,7 @@ func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*d } func (n *layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error { - return n.deleteSystemObject(ctx, bktInfo, bktInfo.CORSObjectName()) + return n.DeleteSystemObject(ctx, bktInfo, bktInfo.CORSObjectName()) } func checkCORS(cors *data.CORSConfiguration) error { diff --git a/api/layer/layer.go b/api/layer/layer.go index ea589578..b087e0d9 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -352,6 +352,7 @@ type ( Metadata map[string]string Prefix string Reader io.Reader + Lock *data.ObjectLock } // ListObjectVersionsParams stores list objects versions parameters. @@ -398,11 +399,13 @@ type ( DeleteBucket(ctx context.Context, p *DeleteBucketParams) error GetObject(ctx context.Context, p *GetObjectParams) error + HeadSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) (*data.ObjectInfo, error) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) GetObjectTagging(ctx context.Context, p *data.ObjectInfo) (map[string]string, error) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) + PutSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error PutBucketTagging(ctx context.Context, bucket string, tagSet map[string]string) error @@ -413,6 +416,7 @@ type ( ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) ([]*VersionedObject, error) + DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) error DeleteBucketTagging(ctx context.Context, bucket string) error @@ -631,7 +635,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, oi *data.ObjectInfo) (map[ Owner: oi.Owner, } - objInfo, err := n.headSystemObject(ctx, bktInfo, oi.TagsObject()) + objInfo, err := n.HeadSystemObject(ctx, bktInfo, oi.TagsObject()) if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) { return nil, err } @@ -646,7 +650,7 @@ func (n *layer) GetBucketTagging(ctx context.Context, bucketName string) (map[st return nil, err } - objInfo, err := n.headSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName)) + objInfo, err := n.HeadSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName)) if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) { return nil, err } @@ -686,11 +690,8 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error Reader: nil, } - if _, err := n.putSystemObject(ctx, s); err != nil { - return err - } - - return nil + _, err := n.PutSystemObject(ctx, s) + return err } // PutBucketTagging into storage. @@ -708,11 +709,8 @@ func (n *layer) PutBucketTagging(ctx context.Context, bucketName string, tagSet Reader: nil, } - if _, err = n.putSystemObject(ctx, s); err != nil { - return err - } - - return nil + _, err = n.PutSystemObject(ctx, s) + return err } // DeleteObjectTagging from storage. @@ -721,7 +719,7 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) err if err != nil { return err } - return n.deleteSystemObject(ctx, bktInfo, p.TagsObject()) + return n.DeleteSystemObject(ctx, bktInfo, p.TagsObject()) } // DeleteBucketTagging from storage. @@ -731,7 +729,7 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, bucketName string) erro return err } - return n.deleteSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName)) + return n.DeleteSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName)) } // CopyObject from one bucket into another bucket. diff --git a/api/layer/object.go b/api/layer/object.go index 120350e9..873a6104 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -202,6 +202,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjec if p.Lock != nil { // todo form lock system object + // attributes = append(attributes, attributesFromLock(p.Lock)...) } meta, err := n.objectHead(ctx, bkt.CID, id) diff --git a/api/layer/system_object.go b/api/layer/system_object.go index 7f275ae3..01342ed1 100644 --- a/api/layer/system_object.go +++ b/api/layer/system_object.go @@ -14,7 +14,12 @@ import ( "go.uber.org/zap" ) -func (n *layer) putSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) { +const ( + AttributeComplianceMode = ".s3-compliance-mode" + AttributeRetainUntil = ".s3-retain-until" +) + +func (n *layer) PutSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) { objInfo, err := n.putSystemObjectIntoNeoFS(ctx, p) if err != nil { return nil, err @@ -27,7 +32,7 @@ func (n *layer) putSystemObject(ctx context.Context, p *PutSystemObjectParams) ( return objInfo, nil } -func (n *layer) headSystemObject(ctx context.Context, bkt *data.BucketInfo, objName string) (*data.ObjectInfo, error) { +func (n *layer) HeadSystemObject(ctx context.Context, bkt *data.BucketInfo, objName string) (*data.ObjectInfo, error) { if objInfo := n.systemCache.GetObject(systemObjectKey(bkt, objName)); objInfo != nil { return objInfo, nil } @@ -44,7 +49,7 @@ func (n *layer) headSystemObject(ctx context.Context, bkt *data.BucketInfo, objN return versions.getLast(), nil } -func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error { +func (n *layer) DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error { f := &findParams{ attr: [2]string{objectSystemAttributeName, name}, cid: bktInfo.CID, @@ -92,6 +97,12 @@ func (n *layer) putSystemObjectIntoNeoFS(ctx context.Context, p *PutSystemObject v = tagEmptyMark } + if p.Lock != nil { + // todo form lock system object + + prm.Attributes = append(prm.Attributes, attributesFromLock(p.Lock)...) + } + prm.Attributes = append(prm.Attributes, [2]string{k, v}) } @@ -262,3 +273,21 @@ func (n *layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) err return nil } + +func attributesFromLock(lock *data.ObjectLock) []*object.Attribute { + var result []*object.Attribute + if !lock.LegalHold { + attrRetainUntil := object.NewAttribute() + attrRetainUntil.SetKey(AttributeRetainUntil) + attrRetainUntil.SetValue(lock.Until.Format(time.RFC3339)) + result = append(result, attrRetainUntil) + if lock.IsCompliance { + attrCompliance := object.NewAttribute() + attrCompliance.SetKey(AttributeComplianceMode) + attrCompliance.SetValue(strconv.FormatBool(true)) + result = append(result, attrCompliance) + } + } + + return result +} diff --git a/api/layer/versioning.go b/api/layer/versioning.go index af7c2862..f5cf77d7 100644 --- a/api/layer/versioning.go +++ b/api/layer/versioning.go @@ -399,7 +399,7 @@ func contains(list []string, elem string) bool { } func (n *layer) getBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) { - objInfo, err := n.headSystemObject(ctx, bktInfo, bktInfo.SettingsObjectName()) + objInfo, err := n.HeadSystemObject(ctx, bktInfo, bktInfo.SettingsObjectName()) if err != nil { return nil, err }