[#357] Refactor delete objects

Signed-off-by: Angira Kekteeva <kira@nspcc.ru>
This commit is contained in:
Angira Kekteeva 2022-03-31 10:24:29 +04:00 committed by Alex Vanin
parent b7aac223df
commit 021f5d4dd0
4 changed files with 78 additions and 25 deletions

View file

@ -7,8 +7,10 @@ import (
"strings" "strings"
"github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/nspcc-dev/neofs-s3-gw/api/layer"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap" "go.uber.org/zap"
"go.uber.org/zap/zapcore" "go.uber.org/zap/zapcore"
) )
@ -66,7 +68,18 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
deletedObjects, err := h.obj.DeleteObjects(r.Context(), bktInfo, versionedObject) bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
return
}
p := &layer.DeleteObjectParams{
BktInfo: bktInfo,
BktSettings: bktSettings,
Objects: versionedObject,
}
deletedObjects, err := h.obj.DeleteObjects(r.Context(), p)
deletedObject := deletedObjects[0] deletedObject := deletedObjects[0]
if err == nil { if err == nil {
err = deletedObject.Error err = deletedObject.Error
@ -145,7 +158,18 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
return nil return nil
}) })
deletedObjects, err := h.obj.DeleteObjects(r.Context(), bktInfo, toRemove) bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
return
}
p := &layer.DeleteObjectParams{
BktInfo: bktInfo,
BktSettings: bktSettings,
Objects: toRemove,
}
deletedObjects, err := h.obj.DeleteObjects(r.Context(), p)
if !requested.Quiet && err != nil { if !requested.Quiet && err != nil {
h.logAndSendError(w, "couldn't delete objects", reqInfo, err) h.logAndSendError(w, "couldn't delete objects", reqInfo, err)
return return

View file

@ -436,7 +436,17 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
h.log.Error("couldn't send notification: %w", zap.Error(err)) h.log.Error("couldn't send notification: %w", zap.Error(err))
} }
if _, err = h.obj.DeleteObjects(r.Context(), bktInfo, []*layer.VersionedObject{{Name: initPart.Name}}); err != nil { bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
}
p := &layer.DeleteObjectParams{
BktInfo: bktInfo,
BktSettings: bktSettings,
Objects: []*layer.VersionedObject{{Name: initPart.Name}},
}
if _, err = h.obj.DeleteObjects(r.Context(), p); err != nil {
h.logAndSendError(w, "could not delete init file of multipart upload", reqInfo, err, additional...) h.logAndSendError(w, "could not delete init file of multipart upload", reqInfo, err, additional...)
return return
} }
@ -447,9 +457,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
Key: objInfo.Name, Key: objInfo.Name,
} }
if settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo); err != nil { if bktSettings != nil && bktSettings.VersioningEnabled {
h.log.Warn("couldn't get bucket versioning", zap.String("bucket name", reqInfo.BucketName), zap.Error(err))
} else if settings.VersioningEnabled {
w.Header().Set(api.AmzVersionID, objInfo.Version()) w.Header().Set(api.AmzVersionID, objInfo.Version())
} }

View file

@ -107,6 +107,12 @@ type (
Lock *data.ObjectLock Lock *data.ObjectLock
} }
DeleteObjectParams struct {
BktInfo *data.BucketInfo
BktSettings *data.BucketSettings
Objects []*VersionedObject
}
// PutSettingsParams stores object copy request parameters. // PutSettingsParams stores object copy request parameters.
PutSettingsParams struct { PutSettingsParams struct {
BktInfo *data.BucketInfo BktInfo *data.BucketInfo
@ -220,7 +226,7 @@ type (
ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error)
ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
DeleteObjects(ctx context.Context, bktInfo *data.BucketInfo, objects []*VersionedObject) ([]*VersionedObject, error) DeleteObjects(ctx context.Context, p *DeleteObjectParams) ([]*VersionedObject, error)
DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error
DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) error DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) error
DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error
@ -536,12 +542,18 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.Obje
} }
// DeleteObject removes all objects with passed nice name. // DeleteObject removes all objects with passed nice name.
func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) *VersionedObject { func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings *data.BucketSettings, obj *VersionedObject) *VersionedObject {
var ( var (
err error err error
ids []*oid.ID ids []*oid.ID
versioningEnabled = false
) )
if settings != nil {
versioningEnabled = settings.VersioningEnabled
}
p := &PutObjectParams{ p := &PutObjectParams{
BktInfo: bkt, BktInfo: bkt,
Object: obj.Name, Object: obj.Name,
@ -549,8 +561,6 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, obj *Ver
Header: map[string]string{}, Header: map[string]string{},
} }
versioningEnabled := n.isVersioningEnabled(ctx, bkt)
// Current implementation doesn't consider "unversioned" mode (so any deletion creates "delete-mark" object). // Current implementation doesn't consider "unversioned" mode (so any deletion creates "delete-mark" object).
// The reason is difficulties to determinate whether versioning mode is "unversioned" or "suspended". // The reason is difficulties to determinate whether versioning mode is "unversioned" or "suspended".
@ -613,12 +623,12 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, obj *Ver
} }
// DeleteObjects from the storage. // DeleteObjects from the storage.
func (n *layer) DeleteObjects(ctx context.Context, bktInfo *data.BucketInfo, objects []*VersionedObject) ([]*VersionedObject, error) { func (n *layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) ([]*VersionedObject, error) {
for i, obj := range objects { for i, obj := range p.Objects {
objects[i] = n.deleteObject(ctx, bktInfo, obj) p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.BktSettings, obj)
} }
return objects, nil return p.Objects, nil
} }
func (n *layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) { func (n *layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {

View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-s3-gw/internal/neofstest" "github.com/nspcc-dev/neofs-s3-gw/internal/neofstest"
@ -58,10 +59,15 @@ func (tc *testContext) getObject(objectName, versionID string, needError bool) (
return objInfo, content.Bytes() return objInfo, content.Bytes()
} }
func (tc *testContext) deleteObject(objectName, versionID string) { func (tc *testContext) deleteObject(objectName, versionID string, settings *data.BucketSettings) {
deletedObjects, err := tc.layer.DeleteObjects(tc.ctx, tc.bktInfo, []*VersionedObject{ p := &DeleteObjectParams{
BktInfo: tc.bktInfo,
BktSettings: settings,
Objects: []*VersionedObject{
{Name: objectName, VersionID: versionID}, {Name: objectName, VersionID: versionID},
}) },
}
deletedObjects, err := tc.layer.DeleteObjects(tc.ctx, p)
require.NoError(tc.t, err) require.NoError(tc.t, err)
for _, obj := range deletedObjects { for _, obj := range deletedObjects {
require.NoError(tc.t, obj.Error) require.NoError(tc.t, obj.Error)
@ -223,16 +229,17 @@ func TestSimpleNoVersioning(t *testing.T) {
func TestVersioningDeleteObject(t *testing.T) { func TestVersioningDeleteObject(t *testing.T) {
tc := prepareContext(t) tc := prepareContext(t)
settings := &data.BucketSettings{VersioningEnabled: true}
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{ err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
BktInfo: tc.bktInfo, BktInfo: tc.bktInfo,
Settings: &data.BucketSettings{VersioningEnabled: true}, Settings: settings,
}) })
require.NoError(t, err) require.NoError(t, err)
tc.putObject([]byte("content obj1 v1")) tc.putObject([]byte("content obj1 v1"))
tc.putObject([]byte("content obj1 v2")) tc.putObject([]byte("content obj1 v2"))
tc.deleteObject(tc.obj, "") tc.deleteObject(tc.obj, "", settings)
tc.getObject(tc.obj, "", true) tc.getObject(tc.obj, "", true)
tc.checkListObjects() tc.checkListObjects()
@ -240,9 +247,10 @@ func TestVersioningDeleteObject(t *testing.T) {
func TestVersioningDeleteSpecificObjectVersion(t *testing.T) { func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
tc := prepareContext(t) tc := prepareContext(t)
settings := &data.BucketSettings{VersioningEnabled: true}
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{ err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
BktInfo: tc.bktInfo, BktInfo: tc.bktInfo,
Settings: &data.BucketSettings{VersioningEnabled: true}, Settings: settings,
}) })
require.NoError(t, err) require.NoError(t, err)
@ -251,18 +259,18 @@ func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
objV3Content := []byte("content obj1 v3") objV3Content := []byte("content obj1 v3")
objV3Info := tc.putObject(objV3Content) objV3Info := tc.putObject(objV3Content)
tc.deleteObject(tc.obj, objV2Info.Version()) tc.deleteObject(tc.obj, objV2Info.Version(), settings)
tc.getObject(tc.obj, objV2Info.Version(), true) tc.getObject(tc.obj, objV2Info.Version(), true)
_, buffer3 := tc.getObject(tc.obj, "", false) _, buffer3 := tc.getObject(tc.obj, "", false)
require.Equal(t, objV3Content, buffer3) require.Equal(t, objV3Content, buffer3)
tc.deleteObject(tc.obj, "") tc.deleteObject(tc.obj, "", settings)
tc.getObject(tc.obj, "", true) tc.getObject(tc.obj, "", true)
for _, ver := range tc.listVersions().DeleteMarker { for _, ver := range tc.listVersions().DeleteMarker {
if ver.IsLatest { if ver.IsLatest {
tc.deleteObject(tc.obj, ver.Object.Version()) tc.deleteObject(tc.obj, ver.Object.Version(), settings)
} }
} }
@ -351,7 +359,10 @@ func TestNoVersioningDeleteObject(t *testing.T) {
tc.putObject([]byte("content obj1 v1")) tc.putObject([]byte("content obj1 v1"))
tc.putObject([]byte("content obj1 v2")) tc.putObject([]byte("content obj1 v2"))
tc.deleteObject(tc.obj, "") versioning, err := tc.layer.GetBucketSettings(tc.ctx, tc.bktInfo)
require.Error(t, err, errors.GetAPIError(errors.ErrNoSuchKey))
tc.deleteObject(tc.obj, "", versioning)
tc.getObject(tc.obj, "", true) tc.getObject(tc.obj, "", true)
tc.checkListObjects() tc.checkListObjects()
} }