From aec4f54a00feaee10e1dbaf1eab531dc6c4599e2 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 28 May 2022 12:39:37 +0300 Subject: [PATCH] [#1444] pilorama: Optimize internal encoding/decoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` name old time/op new time/op delta ApplySequential/bbolt-8 55.5µs ± 4% 55.5µs ± 3% ~ (p=1.000 n=10+7) ApplyReorderLast/bbolt-8 108µs ± 6% 112µs ± 8% ~ (p=0.077 n=9+9) name old alloc/op new alloc/op delta ApplySequential/bbolt-8 28.8kB ± 3% 27.7kB ± 6% -3.79% (p=0.005 n=10+10) ApplyReorderLast/bbolt-8 41.4kB ± 5% 38.9kB ± 5% -6.19% (p=0.001 n=10+9) name old allocs/op new allocs/op delta ApplySequential/bbolt-8 262 ± 2% 235 ±10% -10.41% (p=0.000 n=10+10) ApplyReorderLast/bbolt-8 684 ± 6% 616 ± 7% -10.04% (p=0.000 n=10+9) ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/boltdb.go | 36 +++++----- pkg/local_object_storage/pilorama/meta.go | 80 +++++++++++++++------ 2 files changed, 78 insertions(+), 38 deletions(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index e1444aab8..c582fc6e9 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -231,9 +231,13 @@ func (t *boltForest) applyOperation(logBucket, treeBucket *bbolt.Bucket, m *Move key, value := c.Last() + b := bytes.NewReader(nil) + r := io.NewBinReaderFromIO(b) + // 1. Undo up until the desired timestamp is here. for len(key) == 8 && binary.BigEndian.Uint64(key) > m.Time { - if err := t.logFromBytes(&tmp, value); err != nil { + b.Reset(value) + if err := t.logFromBytes(&tmp, r); err != nil { return nil, err } if err := t.undo(&tmp.Move, &tmp, treeBucket, cKey[:]); err != nil { @@ -253,7 +257,8 @@ func (t *boltForest) applyOperation(logBucket, treeBucket *bbolt.Bucket, m *Move // 3. Re-apply all other operations. for len(key) == 8 { - if err := t.logFromBytes(&tmp, value); err != nil { + b.Reset(value) + if err := t.logFromBytes(&tmp, r); err != nil { return nil, err } if err := t.do(logBucket, treeBucket, cKey[:], &tmp); err != nil { @@ -559,40 +564,37 @@ func (t *boltForest) moveFromBytes(m *Move, data []byte) error { r := io.NewBinReaderFromBuf(data) m.Child = r.ReadU64LE() m.Parent = r.ReadU64LE() - if err := m.Meta.FromBytes(r.ReadVarBytes()); err != nil { - return err - } + m.Meta.DecodeBinary(r) return r.Err } -func (t *boltForest) logFromBytes(lm *LogMove, data []byte) error { - r := io.NewBinReaderFromBuf(data) +func (t *boltForest) logFromBytes(lm *LogMove, r *io.BinReader) error { lm.Child = r.ReadU64LE() lm.Parent = r.ReadU64LE() - if err := lm.Meta.FromBytes(r.ReadVarBytes()); err != nil { - return err - } - + lm.Meta.DecodeBinary(r) lm.HasOld = r.ReadBool() if lm.HasOld { lm.Old.Parent = r.ReadU64LE() - if err := lm.Old.Meta.FromBytes(r.ReadVarBytes()); err != nil { - return err - } + lm.Old.Meta.DecodeBinary(r) } - return r.Err } func (t *boltForest) logToBytes(lm *LogMove) []byte { w := io.NewBufBinWriter() + size := 8 + 8 + lm.Meta.Size() + 1 + if lm.HasOld { + size += 8 + lm.Old.Meta.Size() + } + + w.Grow(size) w.WriteU64LE(lm.Child) w.WriteU64LE(lm.Parent) - w.WriteVarBytes(lm.Meta.Bytes()) + lm.Meta.EncodeBinary(w.BinWriter) w.WriteBool(lm.HasOld) if lm.HasOld { w.WriteU64LE(lm.Old.Parent) - w.WriteVarBytes(lm.Old.Meta.Bytes()) + lm.Old.Meta.EncodeBinary(w.BinWriter) } return w.Bytes() } diff --git a/pkg/local_object_storage/pilorama/meta.go b/pkg/local_object_storage/pilorama/meta.go index f5ed21be7..49b7f6477 100644 --- a/pkg/local_object_storage/pilorama/meta.go +++ b/pkg/local_object_storage/pilorama/meta.go @@ -10,31 +10,13 @@ func (x *Meta) FromBytes(data []byte) error { } r := io.NewBinReaderFromBuf(data) - ts := r.ReadVarUint() - size := r.ReadVarUint() - m := make([]KeyValue, size) - for i := range m { - m[i].Key = r.ReadString() - m[i].Value = r.ReadVarBytes() - } - if r.Err != nil { - return r.Err - } - - x.Time = ts - x.Items = m - return nil + x.DecodeBinary(r) + return r.Err } func (x Meta) Bytes() []byte { w := io.NewBufBinWriter() - w.WriteVarUint(x.Time) - w.WriteVarUint(uint64(len(x.Items))) - for _, e := range x.Items { - w.WriteString(e.Key) - w.WriteVarBytes(e.Value) - } - + x.EncodeBinary(w.BinWriter) return w.Bytes() } @@ -46,3 +28,59 @@ func (x Meta) GetAttr(name string) []byte { } return nil } + +// DecodeBinary implements the io.Serializable interface. +func (x *Meta) DecodeBinary(r *io.BinReader) { + ts := r.ReadVarUint() + size := r.ReadVarUint() + m := make([]KeyValue, size) + for i := range m { + m[i].Key = r.ReadString() + m[i].Value = r.ReadVarBytes() + } + if r.Err != nil { + return + } + + x.Time = ts + x.Items = m +} + +// EncodeBinary implements the io.Serializable interface. +func (x Meta) EncodeBinary(w *io.BinWriter) { + w.WriteVarUint(x.Time) + w.WriteVarUint(uint64(len(x.Items))) + for _, e := range x.Items { + w.WriteString(e.Key) + w.WriteVarBytes(e.Value) + } +} + +// Size returns size of x in bytes. +func (x Meta) Size() int { + size := getVarIntSize(x.Time) + size += getVarIntSize(uint64(len(x.Items))) + for i := range x.Items { + ln := len(x.Items[i].Key) + size += getVarIntSize(uint64(ln)) + ln + + ln = len(x.Items[i].Value) + size += getVarIntSize(uint64(ln)) + ln + } + return size +} + +// getVarIntSize returns the size in number of bytes of a variable integer. +// (reference: GetVarSize(int value), https://github.com/neo-project/neo/blob/master/neo/IO/Helper.cs) +func getVarIntSize(value uint64) int { + var size int + + if value < 0xFD { + size = 1 // unit8 + } else if value <= 0xFFFF { + size = 3 // byte + uint16 + } else { + size = 5 // byte + uint32 + } + return size +}