[#195] Implement PUT, GET locks to certain object

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-02-28 16:39:04 +03:00 committed by Angira Kekteeva
parent 8553158b81
commit 7d6271be8a
10 changed files with 337 additions and 36 deletions

View file

@ -104,3 +104,9 @@ func (o *ObjectInfo) Address() *address.Address {
// TagsObject returns name of system object for tags.
func (o *ObjectInfo) TagsObject() string { return ".tagset." + o.Name + "." + o.Version() }
// LegalHoldObject returns name of system object for lock object.
func (o *ObjectInfo) LegalHoldObject() string { return ".lock." + o.Name + "." + o.Version() }
// RetentionObject returns name of system object for retention lock object.
func (o *ObjectInfo) RetentionObject() string { return ".retention." + o.Name + "." + o.Version() }

View file

@ -1,9 +1,13 @@
package data
import "time"
import (
"encoding/xml"
"time"
)
type (
ObjectLockConfiguration struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ObjectLockConfiguration" json:"-"`
ObjectLockEnabled string `xml:"ObjectLockEnabled" json:"ObjectLockEnabled"`
Rule *ObjectLockRule `xml:"Rule" json:"Rule"`
}
@ -18,6 +22,17 @@ type (
Years int64 `xml:"Years" json:"Years"`
}
LegalHold struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LegalHold" json:"-"`
Status string `xml:"Status" json:"Status"`
}
Retention struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Retention" json:"-"`
Mode string `xml:"Mode" json:"Mode"`
RetainUntilDate string `xml:"RetainUntilDate" json:"RetainUntilDate"`
}
ObjectLock struct {
Until time.Time
LegalHold bool

View file

@ -4,6 +4,7 @@ import (
"encoding/xml"
"fmt"
"net/http"
"strconv"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
@ -20,6 +21,7 @@ const (
governanceMode = "GOVERNANCE"
complianceMode = "COMPLIANCE"
legalHoldOn = "ON"
legalHoldOff = "OFF"
)
func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
@ -108,6 +110,240 @@ func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
}
}
func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil {
h.logAndSendError(w, "expected owner doesn't match", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "object lock disabled", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound))
return
}
legalHold := &data.LegalHold{}
if err = xml.NewDecoder(r.Body).Decode(legalHold); err != nil {
h.logAndSendError(w, "couldn't parse legal hold configuration", reqInfo, err)
return
}
if legalHold.Status != legalHoldOn && legalHold.Status != legalHoldOff {
h.logAndSendError(w, "invalid legal hold status", reqInfo,
fmt.Errorf("invalid status %s", legalHold.Status))
return
}
p := &layer.HeadObjectParams{
Bucket: reqInfo.BucketName,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), p)
if err != nil {
h.logAndSendError(w, "could not get object info", reqInfo, err)
return
}
lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject())
if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
return
}
if lockInfo == nil && legalHold.Status == legalHoldOff ||
lockInfo != nil && legalHold.Status == legalHoldOn {
return
}
if lockInfo != nil {
if err = h.obj.DeleteSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject()); err != nil {
h.logAndSendError(w, "couldn't delete legal hold", reqInfo, err)
return
}
} else {
ps := &layer.PutSystemObjectParams{
BktInfo: bktInfo,
ObjName: objInfo.LegalHoldObject(),
Lock: &data.ObjectLock{LegalHold: true},
}
if _, err = h.obj.PutSystemObject(r.Context(), ps); err != nil {
h.logAndSendError(w, "couldn't put legal hold", reqInfo, err)
return
}
}
}
func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil {
h.logAndSendError(w, "expected owner doesn't match", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "object lock disabled", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound))
return
}
p := &layer.HeadObjectParams{
Bucket: reqInfo.BucketName,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), p)
if err != nil {
h.logAndSendError(w, "could not get object info", reqInfo, err)
return
}
lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.LegalHoldObject())
if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
return
}
legalHold := &data.LegalHold{Status: legalHoldOff}
if lockInfo != nil {
legalHold.Status = legalHoldOn
}
if err = api.EncodeToResponse(w, legalHold); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
}
}
func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil {
h.logAndSendError(w, "expected owner doesn't match", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "object lock disabled", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound))
return
}
retention := &data.Retention{}
if err = xml.NewDecoder(r.Body).Decode(retention); err != nil {
h.logAndSendError(w, "couldn't parse object retention", reqInfo, err)
return
}
lock, err := formObjectLockFromRetention(retention, r.Header)
if err != nil {
h.logAndSendError(w, "invalid retention configuration", reqInfo, err)
return
}
p := &layer.HeadObjectParams{
Bucket: reqInfo.BucketName,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), p)
if err != nil {
h.logAndSendError(w, "could not get object info", reqInfo, err)
return
}
lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.RetentionObject())
if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
return
}
if lockInfo != nil && lockInfo.Headers[layer.AttributeComplianceMode] != "" {
h.logAndSendError(w, "couldn't change compliance lock mode", reqInfo, err)
return
}
ps := &layer.PutSystemObjectParams{
BktInfo: bktInfo,
ObjName: objInfo.RetentionObject(),
Lock: lock,
}
if _, err = h.obj.PutSystemObject(r.Context(), ps); err != nil {
h.logAndSendError(w, "couldn't put legal hold", reqInfo, err)
return
}
}
func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil {
h.logAndSendError(w, "expected owner doesn't match", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "object lock disabled", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound))
return
}
p := &layer.HeadObjectParams{
Bucket: reqInfo.BucketName,
Object: reqInfo.ObjectName,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
objInfo, err := h.obj.GetObjectInfo(r.Context(), p)
if err != nil {
h.logAndSendError(w, "could not get object info", reqInfo, err)
return
}
lockInfo, err := h.obj.HeadSystemObject(r.Context(), bktInfo, objInfo.RetentionObject())
if err != nil {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
return
}
retention := &data.Retention{
Mode: governanceMode,
RetainUntilDate: lockInfo.Headers[layer.AttributeRetainUntil],
}
if lockInfo.Headers[layer.AttributeComplianceMode] != "" {
retention.Mode = complianceMode
}
if err = api.EncodeToResponse(w, retention); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
}
}
func checkLockConfiguration(conf *data.ObjectLockConfiguration) error {
if conf.ObjectLockEnabled != "" && conf.ObjectLockEnabled != enabledValue {
return fmt.Errorf("invalid ObjectLockEnabled value: %s", conf.ObjectLockEnabled)
@ -180,3 +416,34 @@ func existLockHeaders(header http.Header) bool {
header.Get(api.AmzObjectLockLegalHold) != "" ||
header.Get(api.AmzObjectLockRetainUntilDate) != ""
}
func formObjectLockFromRetention(retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
var err error
var bypassGovernance bool
bypass := header.Get(api.AmzBypassGovernanceRetention)
if bypass != "" {
if bypassGovernance, err = strconv.ParseBool(bypass); err != nil {
return nil, fmt.Errorf("couldn't parse '%s' header", api.AmzBypassGovernanceRetention)
}
}
if retention.Mode != governanceMode && retention.Mode != complianceMode {
return nil, fmt.Errorf("invalid retention mode: %s", retention.Mode)
}
retentionDate, err := time.Parse(time.RFC3339, retention.RetainUntilDate)
if err != nil {
return nil, fmt.Errorf("couldn't parse retain until date: %s", retention.RetainUntilDate)
}
lock := &data.ObjectLock{
Until: retentionDate,
IsCompliance: retention.Mode == complianceMode,
}
if !lock.IsCompliance && !bypassGovernance {
return nil, fmt.Errorf("you cannot bypase governance mode")
}
return lock, nil
}

View file

@ -11,22 +11,6 @@ func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Requ
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}

View file

@ -50,6 +50,7 @@ const (
AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold"
AmzObjectLockMode = "X-Amz-Object-Lock-Mode"
AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date"
AmzBypassGovernanceRetention = "X-Amz-Bypass-Governance-Retention"
ContainerID = "X-Container-Id"

View file

@ -72,7 +72,7 @@ func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*d
}
func (n *layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error {
return n.deleteSystemObject(ctx, bktInfo, bktInfo.CORSObjectName())
return n.DeleteSystemObject(ctx, bktInfo, bktInfo.CORSObjectName())
}
func checkCORS(cors *data.CORSConfiguration) error {

View file

@ -352,6 +352,7 @@ type (
Metadata map[string]string
Prefix string
Reader io.Reader
Lock *data.ObjectLock
}
// ListObjectVersionsParams stores list objects versions parameters.
@ -398,11 +399,13 @@ type (
DeleteBucket(ctx context.Context, p *DeleteBucketParams) error
GetObject(ctx context.Context, p *GetObjectParams) error
HeadSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) (*data.ObjectInfo, error)
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error)
GetObjectTagging(ctx context.Context, p *data.ObjectInfo) (map[string]string, error)
GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error)
PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error)
PutSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error)
PutObjectTagging(ctx context.Context, p *PutTaggingParams) error
PutBucketTagging(ctx context.Context, bucket string, tagSet map[string]string) error
@ -413,6 +416,7 @@ type (
ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) ([]*VersionedObject, error)
DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error
DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) error
DeleteBucketTagging(ctx context.Context, bucket string) error
@ -631,7 +635,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, oi *data.ObjectInfo) (map[
Owner: oi.Owner,
}
objInfo, err := n.headSystemObject(ctx, bktInfo, oi.TagsObject())
objInfo, err := n.HeadSystemObject(ctx, bktInfo, oi.TagsObject())
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
return nil, err
}
@ -646,7 +650,7 @@ func (n *layer) GetBucketTagging(ctx context.Context, bucketName string) (map[st
return nil, err
}
objInfo, err := n.headSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName))
objInfo, err := n.HeadSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName))
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
return nil, err
}
@ -686,11 +690,8 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutTaggingParams) error
Reader: nil,
}
if _, err := n.putSystemObject(ctx, s); err != nil {
return err
}
return nil
_, err := n.PutSystemObject(ctx, s)
return err
}
// PutBucketTagging into storage.
@ -708,11 +709,8 @@ func (n *layer) PutBucketTagging(ctx context.Context, bucketName string, tagSet
Reader: nil,
}
if _, err = n.putSystemObject(ctx, s); err != nil {
return err
}
return nil
_, err = n.PutSystemObject(ctx, s)
return err
}
// DeleteObjectTagging from storage.
@ -721,7 +719,7 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectInfo) err
if err != nil {
return err
}
return n.deleteSystemObject(ctx, bktInfo, p.TagsObject())
return n.DeleteSystemObject(ctx, bktInfo, p.TagsObject())
}
// DeleteBucketTagging from storage.
@ -731,7 +729,7 @@ func (n *layer) DeleteBucketTagging(ctx context.Context, bucketName string) erro
return err
}
return n.deleteSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName))
return n.DeleteSystemObject(ctx, bktInfo, formBucketTagObjectName(bucketName))
}
// CopyObject from one bucket into another bucket.

View file

@ -202,6 +202,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjec
if p.Lock != nil {
// todo form lock system object
// attributes = append(attributes, attributesFromLock(p.Lock)...)
}
meta, err := n.objectHead(ctx, bkt.CID, id)

View file

@ -14,7 +14,12 @@ import (
"go.uber.org/zap"
)
func (n *layer) putSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) {
const (
AttributeComplianceMode = ".s3-compliance-mode"
AttributeRetainUntil = ".s3-retain-until"
)
func (n *layer) PutSystemObject(ctx context.Context, p *PutSystemObjectParams) (*data.ObjectInfo, error) {
objInfo, err := n.putSystemObjectIntoNeoFS(ctx, p)
if err != nil {
return nil, err
@ -27,7 +32,7 @@ func (n *layer) putSystemObject(ctx context.Context, p *PutSystemObjectParams) (
return objInfo, nil
}
func (n *layer) headSystemObject(ctx context.Context, bkt *data.BucketInfo, objName string) (*data.ObjectInfo, error) {
func (n *layer) HeadSystemObject(ctx context.Context, bkt *data.BucketInfo, objName string) (*data.ObjectInfo, error) {
if objInfo := n.systemCache.GetObject(systemObjectKey(bkt, objName)); objInfo != nil {
return objInfo, nil
}
@ -44,7 +49,7 @@ func (n *layer) headSystemObject(ctx context.Context, bkt *data.BucketInfo, objN
return versions.getLast(), nil
}
func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error {
func (n *layer) DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error {
f := &findParams{
attr: [2]string{objectSystemAttributeName, name},
cid: bktInfo.CID,
@ -92,6 +97,12 @@ func (n *layer) putSystemObjectIntoNeoFS(ctx context.Context, p *PutSystemObject
v = tagEmptyMark
}
if p.Lock != nil {
// todo form lock system object
prm.Attributes = append(prm.Attributes, attributesFromLock(p.Lock)...)
}
prm.Attributes = append(prm.Attributes, [2]string{k, v})
}
@ -262,3 +273,21 @@ func (n *layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) err
return nil
}
func attributesFromLock(lock *data.ObjectLock) []*object.Attribute {
var result []*object.Attribute
if !lock.LegalHold {
attrRetainUntil := object.NewAttribute()
attrRetainUntil.SetKey(AttributeRetainUntil)
attrRetainUntil.SetValue(lock.Until.Format(time.RFC3339))
result = append(result, attrRetainUntil)
if lock.IsCompliance {
attrCompliance := object.NewAttribute()
attrCompliance.SetKey(AttributeComplianceMode)
attrCompliance.SetValue(strconv.FormatBool(true))
result = append(result, attrCompliance)
}
}
return result
}

View file

@ -399,7 +399,7 @@ func contains(list []string, elem string) bool {
}
func (n *layer) getBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
objInfo, err := n.headSystemObject(ctx, bktInfo, bktInfo.SettingsObjectName())
objInfo, err := n.HeadSystemObject(ctx, bktInfo, bktInfo.SettingsObjectName())
if err != nil {
return nil, err
}