From 8307c73fef84f0ee88de17e1c316e7abf4e1637d Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 11 Apr 2024 18:01:05 +0300 Subject: [PATCH] [#364] Fix removing combined object Signed-off-by: Denis Kirillov --- api/handler/multipart_upload_test.go | 30 ++++++++++++++++++++++++++ api/layer/layer.go | 32 ++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/api/handler/multipart_upload_test.go b/api/handler/multipart_upload_test.go index c02220ea..5fdbe243 100644 --- a/api/handler/multipart_upload_test.go +++ b/api/handler/multipart_upload_test.go @@ -38,6 +38,36 @@ func TestMultipartUploadInvalidPart(t *testing.T) { assertS3Error(hc.t, w, s3Errors.GetAPIError(s3Errors.ErrEntityTooSmall)) } +func TestDeleteMultipartAllParts(t *testing.T) { + hc := prepareHandlerContext(t) + + partSize := layer.UploadMinSize + objLen := 6 * partSize + + bktName, bktName2, objName := "bucket", "bucket2", "object" + + // unversioned bucket + createTestBucket(hc, bktName) + multipartUpload(hc, bktName, objName, nil, objLen, partSize) + deleteObject(t, hc, bktName, objName, emptyVersion) + require.Empty(t, hc.tp.Objects()) + + // encrypted multipart + multipartUploadEncrypted(hc, bktName, objName, nil, objLen, partSize) + deleteObject(t, hc, bktName, objName, emptyVersion) + require.Empty(t, hc.tp.Objects()) + + // versions bucket + createTestBucket(hc, bktName2) + putBucketVersioning(t, hc, bktName2, true) + multipartUpload(hc, bktName2, objName, nil, objLen, partSize) + _, hdr := getObject(hc, bktName2, objName) + versionID := hdr.Get("X-Amz-Version-Id") + deleteObject(t, hc, bktName2, objName, emptyVersion) + deleteObject(t, hc, bktName2, objName, versionID) + require.Empty(t, hc.tp.Objects()) +} + func TestMultipartReUploadPart(t *testing.T) { hc := prepareHandlerContext(t) diff --git a/api/layer/layer.go b/api/layer/layer.go index a1d82829..07bc2633 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -4,6 +4,7 @@ import ( "context" "crypto/ecdsa" "crypto/rand" + "encoding/json" "encoding/xml" "fmt" "io" @@ -791,9 +792,40 @@ func (n *layer) removeOldVersion(ctx context.Context, bkt *data.BucketInfo, node return obj.VersionID, nil } + if nodeVersion.IsCombined { + return "", n.removeCombinedObject(ctx, bkt, nodeVersion) + } + return "", n.objectDelete(ctx, bkt, nodeVersion.OID) } +func (n *layer) removeCombinedObject(ctx context.Context, bkt *data.BucketInfo, nodeVersion *data.NodeVersion) error { + combinedObj, err := n.objectGet(ctx, bkt, nodeVersion.OID) + if err != nil { + return fmt.Errorf("get combined object '%s': %w", nodeVersion.OID.EncodeToString(), err) + } + + var parts []*data.PartInfo + if err = json.Unmarshal(combinedObj.Payload(), &parts); err != nil { + return fmt.Errorf("unmarshal combined object parts: %w", err) + } + + for _, part := range parts { + if err = n.objectDelete(ctx, bkt, part.OID); err == nil { + continue + } + + if !client.IsErrObjectAlreadyRemoved(err) && !client.IsErrObjectNotFound(err) { + return fmt.Errorf("couldn't delete part '%s': %w", part.OID.EncodeToString(), err) + } + + n.reqLogger(ctx).Warn(logs.CouldntDeletePart, zap.String("cid", bkt.CID.EncodeToString()), + zap.String("oid", part.OID.EncodeToString()), zap.Int("part number", part.Number), zap.Error(err)) + } + + return n.objectDelete(ctx, bkt, nodeVersion.OID) +} + // DeleteObjects from the storage. func (n *layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject { for i, obj := range p.Objects { diff --git a/go.mod b/go.mod index 38223d8c..7f400e29 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409115729-6eb492025bdd git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240402141549-3790142b10c7 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240410114823-1f190e1668ec + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412102212-530248de754c git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/aws/aws-sdk-go v1.44.6 github.com/bluele/gcache v0.0.2 diff --git a/go.sum b/go.sum index 2dde45b8..ad9faf1c 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240402141549-3790142b10c7 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240402141549-3790142b10c7/go.mod h1:i0RKqiF4z3UOxLSNwhHw+cUz/JyYWuTRpnn9ere4Y3w= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240410114823-1f190e1668ec h1:OG8tBs5CN2HKp10sAWdtiFaX8qSGFyLGWfQmf4FQ6bE= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240410114823-1f190e1668ec/go.mod h1:H/AW85RtYxVTbcgwHW76DqXeKlsiCIOeNXHPqyDBrfQ= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412102212-530248de754c h1:Ei15WKKLDXoFqEIe292szb3RfsyLqRZfeJX2FjFvz6k= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412102212-530248de754c/go.mod h1:H/AW85RtYxVTbcgwHW76DqXeKlsiCIOeNXHPqyDBrfQ= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA=