From 0e9b6be3fdfe58fdfa2f281a3f9f1f15f7bc1ea1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 21 Mar 2022 17:15:09 +0300 Subject: [PATCH] [#1262] metabase: Remove intermediate allocations in `decodeList` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` name old time/op new time/op delta Put/parallel-8 1.57ms ±11% 1.51ms ± 3% -4.06% (p=0.043 n=9+10) Put/sequential-8 5.16ms ± 2% 5.16ms ± 3% ~ (p=1.000 n=9+10) name old alloc/op new alloc/op delta Put/parallel-8 126kB ± 4% 123kB ± 4% -2.54% (p=0.016 n=8+10) Put/sequential-8 171kB ± 1% 170kB ± 1% ~ (p=0.182 n=9+10) name old allocs/op new allocs/op delta Put/parallel-8 565 ± 2% 473 ± 1% -16.18% (p=0.000 n=9+10) Put/sequential-8 819 ± 1% 792 ± 0% -3.34% (p=0.000 n=9+10) ``` Signed-off-by: Evgenii Stratonikov --- .../metabase/index_test.go | 62 +++++++++++++++++++ pkg/local_object_storage/metabase/put.go | 56 ++++++++++++++--- 2 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 pkg/local_object_storage/metabase/index_test.go diff --git a/pkg/local_object_storage/metabase/index_test.go b/pkg/local_object_storage/metabase/index_test.go new file mode 100644 index 000000000..eb7238a59 --- /dev/null +++ b/pkg/local_object_storage/metabase/index_test.go @@ -0,0 +1,62 @@ +package meta + +import ( + "math" + "math/rand" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/stretchr/testify/require" +) + +func Test_getVarUint(t *testing.T) { + data := make([]byte, 10) + for _, val := range []uint64{0, 0xfc, 0xfd, 0xfffe, 0xffff, 0xfffffffe, 0xffffffff, math.MaxUint64} { + expSize := io.PutVarUint(data, val) + actual, actSize, err := getVarUint(data) + require.NoError(t, err) + require.Equal(t, val, actual) + require.Equal(t, expSize, actSize, "value: %x", val) + + _, _, err = getVarUint(data[:expSize-1]) + require.Error(t, err) + } +} + +func Test_decodeList(t *testing.T) { + t.Run("empty", func(t *testing.T) { + lst, err := decodeList(nil) + require.NoError(t, err) + require.True(t, len(lst) == 0) + }) + t.Run("empty, 0 len", func(t *testing.T) { + lst, err := decodeList([]byte{0}) + require.NoError(t, err) + require.True(t, len(lst) == 0) + }) + t.Run("bad len", func(t *testing.T) { + _, err := decodeList([]byte{0xfe}) + require.Error(t, err) + }) + t.Run("random", func(t *testing.T) { + expected := make([][]byte, 20) + for i := range expected { + expected[i] = make([]byte, rand.Uint32()%10) + rand.Read(expected[i]) + } + + data, err := encodeList(expected) + require.NoError(t, err) + + actual, err := decodeList(data) + require.NoError(t, err) + require.Equal(t, expected, actual) + + t.Run("unexpected EOF", func(t *testing.T) { + for i := 1; i < len(data)-1; i++ { + _, err := decodeList(data[:i]) + require.Error(t, err) + } + }) + }) +} diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index e23760a25..aa1b51bb4 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -1,8 +1,10 @@ package meta import ( + "encoding/binary" "errors" "fmt" + gio "io" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neofs-node/pkg/core/object" @@ -377,18 +379,58 @@ func decodeList(data []byte) (lst [][]byte, err error) { if len(data) == 0 { return nil, nil } - r := io.NewBinReaderFromBuf(data) - l := r.ReadVarUint() - lst = make([][]byte, l, l+1) - for i := range lst { - lst[i] = r.ReadVarBytes() + + var offset uint64 + size, n, err := getVarUint(data) + if err != nil { + return nil, err } - if r.Err != nil { - return nil, r.Err + + offset += uint64(n) + lst = make([][]byte, size, size+1) + for i := range lst { + sz, n, err := getVarUint(data[offset:]) + if err != nil { + return nil, err + } + offset += uint64(n) + + next := offset + sz + if uint64(len(data)) < next { + return nil, gio.ErrUnexpectedEOF + } + lst[i] = data[offset:next] + offset = next } return lst, nil } +func getVarUint(data []byte) (uint64, int, error) { + if len(data) == 0 { + return 0, 0, gio.ErrUnexpectedEOF + } + + switch b := data[0]; b { + case 0xfd: + if len(data) < 3 { + return 0, 1, gio.ErrUnexpectedEOF + } + return uint64(binary.LittleEndian.Uint16(data[1:])), 3, nil + case 0xfe: + if len(data) < 5 { + return 0, 1, gio.ErrUnexpectedEOF + } + return uint64(binary.LittleEndian.Uint32(data[1:])), 5, nil + case 0xff: + if len(data) < 9 { + return 0, 1, gio.ErrUnexpectedEOF + } + return binary.LittleEndian.Uint64(data[1:]), 9, nil + default: + return uint64(b), 1, nil + } +} + // updateBlobovniczaID for existing objects if they were moved from from // one blobovnicza to another. func updateBlobovniczaID(tx *bbolt.Tx, addr *addressSDK.Address, id *blobovnicza.ID) error {