[#122] Add delete versioned object

Signed-off-by: Denis Kirillov <denis@nspcc.ru>

api/errors/errors.go

@@ -66,6 +66,7 @@ const (
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNoSuchVersion
+	ErrInvalidVersion
 	ErrNotImplemented
 	ErrPreconditionFailed
 	ErrNotModified
@@ -529,6 +530,12 @@ var errorCodes = errorCodeMap{
 		Description:    "Indicates that the version ID specified in the request does not match an existing version.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrInvalidVersion: {
+		ErrCode:        ErrInvalidVersion,
+		Code:           "InvalidArgument",
+		Description:    "Invalid version id specified",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNotImplemented: {
 		ErrCode:        ErrNotImplemented,
 		Code:           "NotImplemented",

api/handler/delete.go

@@ -8,6 +8,7 @@ import (
 	"github.com/nspcc-dev/neofs-s3-gw/api/errors"
 	"github.com/nspcc-dev/neofs-s3-gw/api/layer"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
 // DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
@@ -21,6 +22,7 @@ type DeleteObjectsRequest struct {
 // ObjectIdentifier carries key name for the object to delete.
 type ObjectIdentifier struct {
 	ObjectName string `xml:"Key"`
+	VersionID  string `xml:"VersionId,omitempty"`
 }
 
 // DeleteError structure.
@@ -43,18 +45,22 @@ type DeleteObjectsResponse struct {
 
 func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
 	reqInfo := api.GetReqInfo(r.Context())
+	versionedObject := []*layer.VersionedObject{{
+		Name:      reqInfo.ObjectName,
+		VersionID: reqInfo.URL.Query().Get("versionId"),
+	}}
 
 	if err := h.checkBucketOwner(r, reqInfo.BucketName); err != nil {
 		h.logAndSendError(w, "expected owner doesn't match", reqInfo, err)
 		return
 	}
 
-	if err := h.obj.DeleteObject(r.Context(), reqInfo.BucketName, reqInfo.ObjectName); err != nil {
+	if errs := h.obj.DeleteObjects(r.Context(), reqInfo.BucketName, versionedObject); len(errs) != 0 && errs[0] != nil {
 		h.log.Error("could not delete object",
 			zap.String("request_id", reqInfo.RequestID),
 			zap.String("bucket_name", reqInfo.BucketName),
 			zap.String("object_name", reqInfo.ObjectName),
-			zap.Error(err))
+			zap.Error(errs[0]))
 
 		// Ignore delete errors:
 
@@ -94,10 +100,14 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	removed := make(map[string]struct{})
-	toRemove := make([]string, 0, len(requested.Objects))
+	toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
 	for _, obj := range requested.Objects {
-		removed[obj.ObjectName] = struct{}{}
+		versionedObj := &layer.VersionedObject{
-		toRemove = append(toRemove, obj.ObjectName)
+			Name:      obj.ObjectName,
+			VersionID: obj.VersionID,
+		}
+		toRemove = append(toRemove, versionedObj)
+		removed[versionedObj.String()] = struct{}{}
 	}
 
 	response := &DeleteObjectsResponse{
@@ -110,9 +120,16 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	marshaler := zapcore.ArrayMarshalerFunc(func(encoder zapcore.ArrayEncoder) error {
+		for _, obj := range toRemove {
+			encoder.AppendString(obj.String())
+		}
+		return nil
+	})
+
 	if errs := h.obj.DeleteObjects(r.Context(), reqInfo.BucketName, toRemove); errs != nil && !requested.Quiet {
 		additional := []zap.Field{
-			zap.Strings("objects_name", toRemove),
+			zap.Array("objects", marshaler),
 			zap.Errors("errors", errs),
 		}
 		h.logAndSendError(w, "could not delete objects", reqInfo, nil, additional...)
@@ -138,7 +155,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	if err := api.EncodeToResponse(w, response); err != nil {
-		h.logAndSendError(w, "could not write response", reqInfo, err, zap.Strings("objects_name", toRemove))
+		h.logAndSendError(w, "could not write response", reqInfo, err, zap.Array("objects", marshaler))
 		return
 	}
 }

api/handler/get.go

@@ -72,7 +72,7 @@ func writeHeaders(h http.Header, info *layer.ObjectInfo) {
 	h.Set(api.LastModified, info.Created.UTC().Format(http.TimeFormat))
 	h.Set(api.ContentLength, strconv.FormatInt(info.Size, 10))
 	h.Set(api.ETag, info.HashSum)
-	h.Set(api.AmzVersionId, info.ID().String())
+	h.Set(api.AmzVersionID, info.ID().String())
 
 	for key, val := range info.Headers {
 		h[api.MetadataPrefix+key] = []string{val}

api/headers.go

@@ -4,7 +4,7 @@ package api
 const (
 	MetadataPrefix       = "X-Amz-Meta-"
 	AmzMetadataDirective = "X-Amz-Metadata-Directive"
-	AmzVersionId   = "X-Amz-Version-Id"
+	AmzVersionID   = "X-Amz-Version-Id"
 
 	LastModified       = "Last-Modified"
 	Date               = "Date"

api/layer/layer.go

@@ -1,6 +1,7 @@
 package layer
 
 import (
+	"bytes"
 	"context"
 	"crypto/ecdsa"
 	"fmt"
@@ -8,6 +9,7 @@ import (
 	"net/url"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
@@ -128,6 +130,12 @@ type (
 		Encode          string
 	}
 
+	// VersionedObject stores object name and version.
+	VersionedObject struct {
+		Name      string
+		VersionID string
+	}
+
 	// NeoFS provides basic NeoFS interface.
 	NeoFS interface {
 		Get(ctx context.Context, address *object.Address) (*object.Object, error)
@@ -158,8 +166,7 @@ type (
 		ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error)
 		ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
 
-		DeleteObject(ctx context.Context, bucket, object string) error
+		DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error
-		DeleteObjects(ctx context.Context, bucket string, objects []string) []error
 	}
 )
 
@@ -168,6 +175,10 @@ const (
 	bktVersionSettingsObject   = ".s3-versioning-settings"
 )
 
+func (t *VersionedObject) String() string {
+	return t.Name + ":" + t.VersionID
+}
+
 // NewLayer creates instance of layer. It checks credentials
 // and establishes gRPC connection with node.
 func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
@@ -330,16 +341,21 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object
 	}
 
 	if len(p.VersionID) == 0 {
-		return n.headLastVersion(ctx, bkt, p.Object)
+		objInfo, err := n.headLastVersion(ctx, bkt, p.Object)
+		if err == nil {
+			if deleteMark, err2 := strconv.ParseBool(objInfo.Headers[versionsDeleteMarkAttr]); err2 == nil && deleteMark {
+				return nil, api.GetAPIError(api.ErrNoSuchKey)
+			}
+		}
+		return objInfo, err
 	}
 
-	return n.headVersion(ctx, bkt, p.Object, p.VersionID)
+	return n.headVersion(ctx, bkt, p.VersionID)
 }
 
 func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) {
 	oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, val: bktVersionSettingsObject})
 	if err != nil {
-		n.log.Error("could not find object id", zap.Error(err))
 		return nil, err
 	}
 
@@ -367,7 +383,12 @@ func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*Ob
 
 // PutObject into storage.
 func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) {
-	return n.objectPut(ctx, p)
+	bkt, err := n.GetBucketInfo(ctx, p.Bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	return n.objectPut(ctx, bkt, p)
 }
 
 // CopyObject from one bucket into another bucket.
@@ -395,35 +416,96 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf
 }
 
 // DeleteObject removes all objects with passed nice name.
-func (n *layer) DeleteObject(ctx context.Context, bucket, filename string) error {
+func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) error {
 	var (
 		err error
 		ids []*object.ID
-		bkt *BucketInfo
 	)
 
-	if bkt, err = n.GetBucketInfo(ctx, bucket); err != nil {
+	versioningEnabled := n.isVersioningEnabled(ctx, bkt)
-		return &errors.DeleteError{
+	if !versioningEnabled && obj.VersionID != "null" && obj.VersionID != "" {
-			Err:    err,
+		return errors.GetAPIError(errors.ErrInvalidVersion)
-			Object: filename,
+	}
+
+	if versioningEnabled {
+		if len(obj.VersionID) != 0 {
+			id := object.NewID()
+			if err := id.Parse(obj.VersionID); err != nil {
+				return &errors.DeleteError{Err: api.GetAPIError(api.ErrInvalidVersion), Object: obj.String()}
+			}
+			ids = []*object.ID{id}
+
+			lastObject, err := n.headLastVersion(ctx, bkt, obj.Name)
+			if err != nil {
+				return &api.DeleteError{Err: err, Object: obj.String()}
+			}
+			if !strings.Contains(lastObject.Headers[versionsAddAttr], obj.VersionID) ||
+				strings.Contains(lastObject.Headers[versionsDelAttr], obj.VersionID) {
+				return &api.DeleteError{Err: api.GetAPIError(api.ErrInvalidVersion), Object: obj.String()}
+			}
+
+			if lastObject.ID().String() == obj.VersionID {
+				if added := lastObject.Headers[versionsAddAttr]; len(added) > 0 {
+					addedVersions := strings.Split(added, ",")
+					sourceCopyVersion, err := n.headVersion(ctx, bkt, addedVersions[len(addedVersions)-1])
+					if err != nil {
+						return &api.DeleteError{Err: err, Object: obj.String()}
+					}
+					p := &CopyObjectParams{
+						SrcObject: sourceCopyVersion,
+						DstBucket: bkt.Name,
+						DstObject: obj.Name,
+						SrcSize:   sourceCopyVersion.Size,
+						Header:    map[string]string{versionsDelAttr: obj.VersionID},
+					}
+					if _, err := n.CopyObject(ctx, p); err != nil {
+						return err
+					}
+				} else {
+					p := &PutObjectParams{
+						Object: obj.Name,
+						Reader: bytes.NewReader(nil),
+						Header: map[string]string{
+							versionsDelAttr:        obj.VersionID,
+							versionsDeleteMarkAttr: strconv.FormatBool(true),
+						},
+					}
+					if _, err := n.objectPut(ctx, bkt, p); err != nil {
+						return &api.DeleteError{Err: err, Object: obj.String()}
+					}
+				}
+			} else {
+				p := &CopyObjectParams{
+					SrcObject: lastObject,
+					DstBucket: bkt.Name,
+					DstObject: obj.Name,
+					SrcSize:   lastObject.Size,
+					Header:    map[string]string{versionsDelAttr: obj.VersionID},
+				}
+				if _, err := n.CopyObject(ctx, p); err != nil {
+					return err
+				}
+			}
+		} else {
+			p := &PutObjectParams{
+				Object: obj.Name,
+				Reader: bytes.NewReader(nil),
+				Header: map[string]string{versionsDeleteMarkAttr: strconv.FormatBool(true)},
+			}
+			if _, err := n.objectPut(ctx, bkt, p); err != nil {
+				return &errors.DeleteError{Err: err, Object: obj.String()}
+			}
 		}
-	} else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, val: filename}); err != nil {
+	} else {
-		return &errors.DeleteError{
+		ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, val: obj.Name})
-			Err:    err,
+		if err != nil {
-			Object: filename,
+			return &errors.DeleteError{Err: err, Object: obj.String()}
 		}
 	}
 
 	for _, id := range ids {
-		addr := object.NewAddress()
+		if err = n.objectDelete(ctx, bkt.CID, id); err != nil {
-		addr.SetObjectID(id)
+			return &errors.DeleteError{Err: err, Object: obj.String()}
-		addr.SetContainerID(bkt.CID)
-
-		if err = n.objectDelete(ctx, addr); err != nil {
-			return &errors.DeleteError{
-				Err:    err,
-				Object: filename,
-			}
 		}
 	}
 
@@ -431,11 +513,16 @@ func (n *layer) DeleteObject(ctx context.Context, bucket, filename string) error
 }
 
 // DeleteObjects from the storage.
-func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []string) []error {
+func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error {
 	var errs = make([]error, 0, len(objects))
 
+	bkt, err := n.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		return append(errs, err)
+	}
+
 	for i := range objects {
-		if err := n.DeleteObject(ctx, bucket, objects[i]); err != nil {
+		if err := n.deleteObject(ctx, bkt, objects[i]); err != nil {
 			errs = append(errs, err)
 		}
 	}
@@ -461,6 +548,14 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
 		return err
 	}
 
+	ids, err := n.objectSearch(ctx, &findParams{cid: bucketInfo.CID})
+	if err != nil {
+		return err
+	}
+	if len(ids) != 0 {
+		return api.GetAPIError(api.ErrBucketNotEmpty)
+	}
+
 	return n.deleteContainer(ctx, bucketInfo.CID)
 }
 
@@ -581,10 +676,7 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams)
 	}
 
 	if objectInfo != nil {
-		addr := object.NewAddress()
+		if err = n.objectDelete(ctx, bucketInfo.CID, objectInfo.ID()); err != nil {
-		addr.SetObjectID(objectInfo.ID())
-		addr.SetContainerID(bucketInfo.CID)
-		if err = n.objectDelete(ctx, addr); err != nil {
 			return nil, err
 		}
 	}

api/layer/object.go

@@ -64,8 +64,9 @@ type (
 )
 
 const (
-	versionsDelAttr = "S3-Versions-del"
+	versionsDelAttr        = "S3-Versions-del"
-	versionsAddAttr = "S3-Versions-add"
+	versionsAddAttr        = "S3-Versions-add"
+	versionsDeleteMarkAttr = "S3-Versions-delete-mark"
 )
 
 // objectSearch returns all available objects by search params.
@@ -121,11 +122,10 @@ func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) {
 }
 
 // objectPut into NeoFS, took payload from io.Reader.
-func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) {
+func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectParams) (*ObjectInfo, error) {
 	var (
 		err error
 		obj string
-		bkt *BucketInfo
 		own = n.Owner(ctx)
 	)
 
@@ -136,9 +136,6 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
 	if obj, err = url.QueryUnescape(p.Object); err != nil {
 		return nil, err
 	}
-	if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
-		return nil, err
-	}
 
 	versioningEnabled := n.isVersioningEnabled(ctx, bkt)
 	lastVersionInfo, err := n.headLastVersion(ctx, bkt, p.Object)
@@ -155,15 +152,18 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
 				versionsAddedStr += ","
 			}
 			versionsAddedStr += lastVersionInfo.ID().String()
-			addedVersions := object.NewAttribute()
+			p.Header[versionsAddAttr] = versionsAddedStr
-			addedVersions.SetKey(versionsAddAttr)
+
-			addedVersions.SetValue(versionsAddedStr)
+			deleted := p.Header[versionsDelAttr]
-			attributes = append(attributes, addedVersions)
+			if delVersions := lastVersionInfo.Headers[versionsDelAttr]; len(delVersions) != 0 {
-			if delVersions := lastVersionInfo.Headers[versionsDelAttr]; len(delVersions) > 0 {
+				if len(deleted) == 0 {
-				deletedVersions := object.NewAttribute()
+					deleted = delVersions
-				deletedVersions.SetKey(versionsDelAttr)
+				} else {
-				deletedVersions.SetValue(delVersions)
+					deleted = delVersions + "," + deleted
-				attributes = append(attributes, deletedVersions)
+				}
+			}
+			if len(deleted) != 0 {
+				p.Header[versionsDelAttr] = deleted
 			}
 		} else {
 			versionsDeletedStr := lastVersionInfo.Headers[versionsDelAttr]
@@ -171,24 +171,19 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
 				versionsDeletedStr += ","
 			}
 			versionsDeletedStr += lastVersionInfo.ID().String()
-			deletedVersions := object.NewAttribute()
+			p.Header[versionsDelAttr] = versionsDeletedStr
-			deletedVersions.SetKey(versionsDelAttr)
-			deletedVersions.SetValue(versionsDeletedStr)
 
-			attributes = append(attributes, deletedVersions)
 			idsToDeleteArr = append(idsToDeleteArr, lastVersionInfo.ID())
 		}
 	}
 
-	unix := strconv.FormatInt(time.Now().UTC().Unix(), 10)
-
 	filename := object.NewAttribute()
 	filename.SetKey(object.AttributeFileName)
 	filename.SetValue(obj)
 
 	createdAt := object.NewAttribute()
 	createdAt.SetKey(object.AttributeTimestamp)
-	createdAt.SetValue(unix)
+	createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10))
 
 	attributes = append(attributes, filename, createdAt)
 
@@ -240,11 +235,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
 	}
 
 	for _, id := range idsToDeleteArr {
-		addr := object.NewAddress()
+		if err = n.objectDelete(ctx, bkt.CID, id); err != nil {
-		addr.SetObjectID(id)
-		addr.SetContainerID(bkt.CID)
-
-		if err = n.objectDelete(ctx, addr); err != nil {
 			n.log.Warn("couldn't delete object",
 				zap.Stringer("version id", id),
 				zap.Error(err))
@@ -284,33 +275,28 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName
 	return objectInfoFromMeta(bkt, infos[len(infos)-1], "", ""), nil
 }
 
-func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, objectName, versionID string) (*ObjectInfo, error) {
+func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID string) (*ObjectInfo, error) {
-	ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName})
+	oid := object.NewID()
-	if err != nil {
+	if err := oid.Parse(versionID); err != nil {
 		return nil, err
 	}
-	if len(ids) == 0 {
-		return nil, api.GetAPIError(api.ErrNoSuchVersion)
-	}
 
-	for _, id := range ids {
+	meta, err := n.objectHead(ctx, bkt.CID, oid)
-		if id.String() == versionID {
+	if err != nil {
-			meta, err := n.objectHead(ctx, bkt.CID, id)
+		if strings.Contains(err.Error(), "not found") {
-			if err != nil {
+			return nil, api.GetAPIError(api.ErrNoSuchVersion)
-				if strings.Contains(err.Error(), "not found") {
-					return nil, api.GetAPIError(api.ErrNoSuchVersion)
-				}
-				return nil, err
-			}
-			return objectInfoFromMeta(bkt, meta, "", ""), nil
 		}
+		return nil, err
 	}
 
-	return nil, api.GetAPIError(api.ErrNoSuchVersion)
+	return objectInfoFromMeta(bkt, meta, "", ""), nil
 }
 
 // objectDelete puts tombstone object into neofs.
-func (n *layer) objectDelete(ctx context.Context, address *object.Address) error {
+func (n *layer) objectDelete(ctx context.Context, cid *cid.ID, oid *object.ID) error {
+	address := object.NewAddress()
+	address.SetContainerID(cid)
+	address.SetObjectID(oid)
 	dop := new(client.DeleteObjectParams)
 	dop.WithAddress(address)
 	n.objCache.Delete(address)

api/layer/util_test.go

@@ -48,6 +48,7 @@ func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *Obje
 		id:          oid,
 		Name:        name,
 		Bucket:      bkt.Name,
+		bucketID:    bkt.CID,
 		Size:        defaultTestPayloadLength,
 		ContentType: defaultTestContentType,
 		Created:     time.Unix(defaultTestCreated.Unix(), 0),