TrueCloudLab/frostfs-sdk-go
33
0
Fork 21
Code Issues 34 Pull requests 1 Releases Activity Actions

[#293] object: Fix payload size limiter

Browse source 
* Sending empty chunks by `writeChunk` should not release new
  objects as this doesn't change `payloadSizeLimiter` internal
  state.
* This also fixes the bug with patcher when an offset of a patch
  equals to `MaxSize` - `payloadSizeLimiter` releases object again
  although state is the same. This led to error because EC-encoder
  receieved empty payload and couldn't not process it.

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
This commit is contained in:
Airat Arifullin 2024-11-05 16:07:40 +03:00 committed by Evgenii Stratonikov
parent 6980651785
commit cb813e27a8
2 changed files with 73 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
object/transformer/transformer.go
View file

@ -261,7 +261,7 @@ func (s *payloadSizeLimiter) initializeLinking(parHdr *object.Object) {
func (s *payloadSizeLimiter) writeChunk(ctx context.Context, chunk []byte) error {
for {
// statement is true if the previous write of bytes reached exactly the boundary.
if s.written > 0 && s.written%s.MaxSize == 0 {
if len(chunk) > 0 && s.written > 0 && s.written%s.MaxSize == 0 {
if s.written == s.MaxSize {
s.prepareFirstChild()
}

72
object/transformer/transformer_test.go
View file

@ -15,6 +15,78 @@ import (
"github.com/stretchr/testify/require"
)
func TestTransformerNonMonotonicWrites(t *testing.T) {
const maxSize = 16
tt := new(testTarget)
target, pk := newPayloadSizeLimiter(maxSize, 0, func() ObjectWriter { return tt })
cnr := cidtest.ID()
hdr := newObject(cnr)
var owner user.ID
user.IDFromKey(&owner, pk.PrivateKey.PublicKey)
hdr.SetOwnerID(owner)
payload := make([]byte, 3*maxSize)
_, _ = rand.Read(payload)
ctx := context.Background()
require.NoError(t, target.WriteHeader(ctx, hdr))
lessThanMaxSizePayload := make([]byte, maxSize/2)
_, _ = rand.Read(lessThanMaxSizePayload)
_, err := target.Write(ctx, lessThanMaxSizePayload)
require.NoError(t, err)
moreThanMaxSizePayload := make([]byte, maxSize*1.5)
_, _ = rand.Read(moreThanMaxSizePayload)
_, err = target.Write(ctx, moreThanMaxSizePayload)
require.NoError(t, err)
_, err = target.Close(ctx)
require.NoError(t, err)
require.Equal(t, 3, len(tt.objects))
require.Equal(t, append(lessThanMaxSizePayload, moreThanMaxSizePayload[:maxSize-len(lessThanMaxSizePayload)]...), tt.objects[0].Payload())
require.Equal(t, moreThanMaxSizePayload[maxSize-len(lessThanMaxSizePayload):], tt.objects[1].Payload())
require.Equal(t, 2, len(tt.objects[2].Children()))
}
func TestTransformerNonEffectiveWrites(t *testing.T) {
const maxSize = 16
tt := new(testTarget)
target, pk := newPayloadSizeLimiter(maxSize, 0, func() ObjectWriter { return tt })
cnr := cidtest.ID()
hdr := newObject(cnr)
var owner user.ID
user.IDFromKey(&owner, pk.PrivateKey.PublicKey)
hdr.SetOwnerID(owner)
payload := make([]byte, 3*maxSize)
_, _ = rand.Read(payload)
ctx := context.Background()
require.NoError(t, target.WriteHeader(ctx, hdr))
_, err := target.Write(ctx, payload)
require.NoError(t, err)
for range 5 {
// run onto unchanged target state
_, err = target.Write(ctx, []byte{})
require.NoError(t, err)
}
_, err = target.Close(ctx)
require.NoError(t, err)
require.Equal(t, 4, len(tt.objects))
require.Equal(t, payload[:maxSize], tt.objects[0].Payload())
require.Equal(t, payload[maxSize:2*maxSize], tt.objects[1].Payload())
require.Equal(t, payload[2*maxSize:], tt.objects[2].Payload())
require.Equal(t, 3, len(tt.objects[3].Children()))
}
func TestTransformer(t *testing.T) {
const maxSize = 100