forked from TrueCloudLab/frostfs-sdk-go
Airat Arifullin
361739e860
* Introduce `patcher` package that contains such interfaces to be
implemented:
- `PatchApplier` - the main patching engine that merges the stream
of patches and the stream of original object payload divided by
ranges. The merged streams result is output to `ChunkedObjectWriter`;
- `RangeProvider` - provides the original object payload by ranges;
- `HeaderProvider` - provides the original object header.
* Introduce `patcher` that implements `PatchApplier`;
* Cover all possible cases with unit-tests.
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
601 lines
16 KiB
Go
601 lines
16 KiB
Go
package patcher
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"testing"
|
|
|
|
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
type mockPatchedObjectWriter struct {
|
|
obj *objectSDK.Object
|
|
}
|
|
|
|
func (m *mockPatchedObjectWriter) Write(_ context.Context, chunk []byte) (int, error) {
|
|
res := append(m.obj.Payload(), chunk...)
|
|
|
|
m.obj.SetPayload(res)
|
|
m.obj.SetPayloadSize(uint64(len(res)))
|
|
|
|
return len(chunk), nil
|
|
}
|
|
|
|
func (m *mockPatchedObjectWriter) WriteHeader(_ context.Context, hdr *objectSDK.Object) error {
|
|
m.obj.ToV2().SetHeader(hdr.ToV2().GetHeader())
|
|
return nil
|
|
}
|
|
|
|
func (m *mockPatchedObjectWriter) Close(context.Context) (*transformer.AccessIdentifiers, error) {
|
|
return &transformer.AccessIdentifiers{}, nil
|
|
}
|
|
|
|
type mockRangeProvider struct {
|
|
originalObjectPayload []byte
|
|
}
|
|
|
|
var _ RangeProvider = (*mockRangeProvider)(nil)
|
|
|
|
func (m *mockRangeProvider) GetRange(_ context.Context, rng *objectSDK.Range) io.Reader {
|
|
offset := rng.GetOffset()
|
|
length := rng.GetLength()
|
|
|
|
if length == 0 {
|
|
return bytes.NewReader(m.originalObjectPayload[offset:])
|
|
}
|
|
return bytes.NewReader(m.originalObjectPayload[offset : offset+length])
|
|
}
|
|
|
|
func newTestObject() (*objectSDK.Object, oid.Address) {
|
|
obj := objectSDK.New()
|
|
|
|
addr := oidtest.Address()
|
|
obj.SetContainerID(addr.Container())
|
|
obj.SetID(addr.Object())
|
|
|
|
return obj, addr
|
|
}
|
|
|
|
func rangeWithOffestWithLength(offset, length uint64) *objectSDK.Range {
|
|
rng := new(objectSDK.Range)
|
|
rng.SetOffset(offset)
|
|
rng.SetLength(length)
|
|
return rng
|
|
}
|
|
|
|
func TestPatchRevert(t *testing.T) {
|
|
obj, _ := newTestObject()
|
|
|
|
modifPatch := &objectSDK.Patch{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, 0),
|
|
|
|
Chunk: []byte("inserted"),
|
|
},
|
|
}
|
|
|
|
originalObjectPayload := []byte("*******************")
|
|
|
|
obj.SetPayload(originalObjectPayload)
|
|
obj.SetPayloadSize(uint64(len(originalObjectPayload)))
|
|
|
|
exp := []byte("inserted*******************")
|
|
|
|
rangeProvider := &mockRangeProvider{
|
|
originalObjectPayload: originalObjectPayload,
|
|
}
|
|
|
|
patchedObj, _ := newTestObject()
|
|
|
|
wr := &mockPatchedObjectWriter{
|
|
obj: patchedObj,
|
|
}
|
|
|
|
prm := Params{
|
|
Header: obj.CutPayload(),
|
|
|
|
RangeProvider: rangeProvider,
|
|
|
|
ObjectWriter: wr,
|
|
}
|
|
|
|
patcher := New(prm)
|
|
|
|
err := patcher.ApplyAttributesPatch(context.Background(), modifPatch.NewAttributes, modifPatch.ReplaceAttributes)
|
|
require.NoError(t, err)
|
|
|
|
err = patcher.ApplyPayloadPatch(context.Background(), modifPatch.PayloadPatch)
|
|
require.NoError(t, err)
|
|
|
|
_, err = patcher.Close(context.Background())
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, exp, patchedObj.Payload())
|
|
|
|
revertPatch := &objectSDK.Patch{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, uint64(len("inserted"))),
|
|
|
|
Chunk: []byte{},
|
|
},
|
|
}
|
|
|
|
rangeProvider = &mockRangeProvider{
|
|
originalObjectPayload: exp,
|
|
}
|
|
|
|
patchedPatchedObj, _ := newTestObject()
|
|
|
|
wr = &mockPatchedObjectWriter{
|
|
obj: patchedPatchedObj,
|
|
}
|
|
|
|
prm = Params{
|
|
Header: patchedObj.CutPayload(),
|
|
|
|
RangeProvider: rangeProvider,
|
|
|
|
ObjectWriter: wr,
|
|
}
|
|
|
|
patcher = New(prm)
|
|
|
|
err = patcher.ApplyAttributesPatch(context.Background(), revertPatch.NewAttributes, revertPatch.ReplaceAttributes)
|
|
require.NoError(t, err)
|
|
|
|
err = patcher.ApplyPayloadPatch(context.Background(), revertPatch.PayloadPatch)
|
|
require.NoError(t, err)
|
|
|
|
_, err = patcher.Close(context.Background())
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, originalObjectPayload, patchedPatchedObj.Payload())
|
|
}
|
|
|
|
func TestPatchRepeatAttributePatch(t *testing.T) {
|
|
obj, _ := newTestObject()
|
|
|
|
modifPatch := &objectSDK.Patch{}
|
|
|
|
originalObjectPayload := []byte("*******************")
|
|
|
|
obj.SetPayload(originalObjectPayload)
|
|
obj.SetPayloadSize(uint64(len(originalObjectPayload)))
|
|
|
|
rangeProvider := &mockRangeProvider{
|
|
originalObjectPayload: originalObjectPayload,
|
|
}
|
|
|
|
patchedObj, _ := newTestObject()
|
|
|
|
wr := &mockPatchedObjectWriter{
|
|
obj: patchedObj,
|
|
}
|
|
|
|
prm := Params{
|
|
Header: obj.CutPayload(),
|
|
|
|
RangeProvider: rangeProvider,
|
|
|
|
ObjectWriter: wr,
|
|
}
|
|
|
|
patcher := New(prm)
|
|
|
|
err := patcher.ApplyAttributesPatch(context.Background(), modifPatch.NewAttributes, modifPatch.ReplaceAttributes)
|
|
require.NoError(t, err)
|
|
|
|
err = patcher.ApplyAttributesPatch(context.Background(), modifPatch.NewAttributes, modifPatch.ReplaceAttributes)
|
|
require.ErrorIs(t, err, ErrAttrPatchAlreadyApplied)
|
|
}
|
|
|
|
func TestPatchEmptyPayloadPatch(t *testing.T) {
|
|
obj, _ := newTestObject()
|
|
|
|
modifPatch := &objectSDK.Patch{}
|
|
|
|
originalObjectPayload := []byte("*******************")
|
|
|
|
obj.SetPayload(originalObjectPayload)
|
|
obj.SetPayloadSize(uint64(len(originalObjectPayload)))
|
|
|
|
rangeProvider := &mockRangeProvider{
|
|
originalObjectPayload: originalObjectPayload,
|
|
}
|
|
|
|
patchedObj, _ := newTestObject()
|
|
|
|
wr := &mockPatchedObjectWriter{
|
|
obj: patchedObj,
|
|
}
|
|
|
|
prm := Params{
|
|
Header: obj.CutPayload(),
|
|
|
|
RangeProvider: rangeProvider,
|
|
|
|
ObjectWriter: wr,
|
|
}
|
|
|
|
patcher := New(prm)
|
|
|
|
err := patcher.ApplyAttributesPatch(context.Background(), modifPatch.NewAttributes, modifPatch.ReplaceAttributes)
|
|
require.NoError(t, err)
|
|
|
|
err = patcher.ApplyPayloadPatch(context.Background(), nil)
|
|
require.ErrorIs(t, err, ErrPayloadPatchIsNil)
|
|
}
|
|
|
|
func newTestAttribute(key, val string) objectSDK.Attribute {
|
|
var attr objectSDK.Attribute
|
|
attr.SetKey(key)
|
|
attr.SetValue(val)
|
|
return attr
|
|
}
|
|
|
|
func TestPatch(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
patches []objectSDK.Patch
|
|
originalObjectPayload []byte
|
|
patchedPayload []byte
|
|
originalHeaderAttributes []objectSDK.Attribute
|
|
patchedHeaderAttributes []objectSDK.Attribute
|
|
expectedPayloadPatchErr error
|
|
}{
|
|
{
|
|
name: "invalid offset",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(100, 0),
|
|
Chunk: []byte(""),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
expectedPayloadPatchErr: ErrOffsetExceedsSize,
|
|
},
|
|
{
|
|
name: "invalid following patch offset",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(10, 0),
|
|
Chunk: []byte(""),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(7, 0),
|
|
Chunk: []byte(""),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
expectedPayloadPatchErr: ErrInvalidPatchOffsetOrder,
|
|
},
|
|
{
|
|
name: "only header patch",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
NewAttributes: []objectSDK.Attribute{
|
|
newTestAttribute("key1", "val2"),
|
|
newTestAttribute("key2", "val2"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedHeaderAttributes: []objectSDK.Attribute{
|
|
newTestAttribute("key1", "val2"),
|
|
newTestAttribute("key2", "val2"),
|
|
},
|
|
},
|
|
{
|
|
name: "header and payload",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
NewAttributes: []objectSDK.Attribute{
|
|
newTestAttribute("key1", "val2"),
|
|
newTestAttribute("key2", "val2"),
|
|
},
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, 0),
|
|
Chunk: []byte("inserted at the beginning"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("inserted at the beginning0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedHeaderAttributes: []objectSDK.Attribute{
|
|
newTestAttribute("key1", "val2"),
|
|
newTestAttribute("key2", "val2"),
|
|
},
|
|
},
|
|
{
|
|
name: "header only merge attributes",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
NewAttributes: []objectSDK.Attribute{
|
|
newTestAttribute("key1", "val2"),
|
|
newTestAttribute("key2", "val2-incoming"),
|
|
},
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, 0),
|
|
Chunk: []byte("inserted at the beginning"),
|
|
},
|
|
},
|
|
},
|
|
originalHeaderAttributes: []objectSDK.Attribute{
|
|
newTestAttribute("key2", "to be popped out"),
|
|
newTestAttribute("key3", "val3"),
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("inserted at the beginning0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedHeaderAttributes: []objectSDK.Attribute{
|
|
newTestAttribute("key2", "val2-incoming"),
|
|
newTestAttribute("key3", "val3"),
|
|
newTestAttribute("key1", "val2"),
|
|
},
|
|
},
|
|
{
|
|
name: "header only then payload",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
NewAttributes: []objectSDK.Attribute{
|
|
newTestAttribute("key1", "val2"),
|
|
newTestAttribute("key2", "val2"),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, 0),
|
|
Chunk: []byte("inserted at the beginning"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("inserted at the beginning0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedHeaderAttributes: []objectSDK.Attribute{
|
|
newTestAttribute("key1", "val2"),
|
|
newTestAttribute("key2", "val2"),
|
|
},
|
|
},
|
|
{
|
|
name: "no effect",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, 0),
|
|
Chunk: []byte(""),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(12, 0),
|
|
Chunk: []byte(""),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(20, 0),
|
|
Chunk: []byte(""),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
},
|
|
{
|
|
name: "insert prefix",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, 0),
|
|
Chunk: []byte("inserted at the beginning"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("inserted at the beginning0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
},
|
|
{
|
|
name: "insert in the middle",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(5, 0),
|
|
Chunk: []byte("inserted somewhere in the middle"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("01234inserted somewhere in the middle56789qwertyuiopasdfghjklzxcvbnm"),
|
|
},
|
|
{
|
|
name: "insert at the end",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(36, 0),
|
|
Chunk: []byte("inserted somewhere at the end"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnminserted somewhere at the end"),
|
|
},
|
|
{
|
|
name: "replace by range",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, 12),
|
|
Chunk: []byte("just replace"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("just replaceertyuiopasdfghjklzxcvbnm"),
|
|
},
|
|
{
|
|
name: "replace and insert some bytes",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, 11),
|
|
Chunk: []byte("replace and append in the middle"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("replace and append in the middlewertyuiopasdfghjklzxcvbnm"),
|
|
},
|
|
{
|
|
name: "replace and insert some bytes in the middle",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(5, 3),
|
|
Chunk: []byte("@@@@@"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("01234@@@@@89qwertyuiopasdfghjklzxcvbnm"),
|
|
},
|
|
{
|
|
name: "a few patches: prefix, suffix",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(0, 0),
|
|
Chunk: []byte("this_will_be_prefix"),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(36, 0),
|
|
Chunk: []byte("this_will_be_suffix"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789qwertyuiopasdfghjklzxcvbnm"),
|
|
patchedPayload: []byte("this_will_be_prefix0123456789qwertyuiopasdfghjklzxcvbnmthis_will_be_suffix"),
|
|
},
|
|
{
|
|
name: "a few patches: replace and insert some bytes",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(10, 3),
|
|
Chunk: []byte("aaaaa"),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(16, 0),
|
|
Chunk: []byte("bbbbb"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("0123456789ABCDEF"),
|
|
patchedPayload: []byte("0123456789aaaaaDEFbbbbb"),
|
|
},
|
|
{
|
|
name: "a few patches: various modifiactions",
|
|
patches: []objectSDK.Patch{
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(4, 8),
|
|
Chunk: []byte("earliest"),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(13, 0),
|
|
Chunk: []byte("known "),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(35, 8),
|
|
Chunk: []byte("a small town"),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(62, 6),
|
|
Chunk: []byte("tablet"),
|
|
},
|
|
},
|
|
{
|
|
PayloadPatch: &objectSDK.PayloadPatch{
|
|
Range: rangeWithOffestWithLength(87, 0),
|
|
Chunk: []byte("Shar-Kali-Sharri"),
|
|
},
|
|
},
|
|
},
|
|
originalObjectPayload: []byte("The ******** mention of Babylon as [insert] appears on a clay ****** from the reign of "),
|
|
patchedPayload: []byte("The earliest known mention of Babylon as a small town appears on a clay tablet from the reign of Shar-Kali-Sharri"),
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
rangeProvider := &mockRangeProvider{
|
|
originalObjectPayload: test.originalObjectPayload,
|
|
}
|
|
|
|
originalObject, _ := newTestObject()
|
|
originalObject.SetPayload(test.originalObjectPayload)
|
|
originalObject.SetPayloadSize(uint64(len(test.originalObjectPayload)))
|
|
originalObject.SetAttributes(test.originalHeaderAttributes...)
|
|
|
|
patchedObject, _ := newTestObject()
|
|
|
|
wr := &mockPatchedObjectWriter{
|
|
obj: patchedObject,
|
|
}
|
|
|
|
prm := Params{
|
|
Header: originalObject.CutPayload(),
|
|
|
|
RangeProvider: rangeProvider,
|
|
|
|
ObjectWriter: wr,
|
|
}
|
|
|
|
patcher := New(prm)
|
|
|
|
for i, patch := range test.patches {
|
|
if i == 0 {
|
|
_ = patcher.ApplyAttributesPatch(context.Background(), patch.NewAttributes, patch.ReplaceAttributes)
|
|
}
|
|
|
|
if patch.PayloadPatch == nil {
|
|
continue
|
|
}
|
|
|
|
err := patcher.ApplyPayloadPatch(context.Background(), patch.PayloadPatch)
|
|
if err != nil && test.expectedPayloadPatchErr != nil {
|
|
require.ErrorIs(t, err, test.expectedPayloadPatchErr)
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
_, err := patcher.Close(context.Background())
|
|
require.NoError(t, err)
|
|
require.Equal(t, test.patchedPayload, patchedObject.Payload())
|
|
|
|
patchedAttrs := append([]objectSDK.Attribute{}, test.patchedHeaderAttributes...)
|
|
require.Equal(t, patchedAttrs, patchedObject.Attributes())
|
|
})
|
|
}
|
|
|
|
}
|