forked from TrueCloudLab/neoneo-go
cd42b8b20c
This simple approach allows to improve the performance of BoltDB and LevelDB in both terms of speed and allocations for retrieving GasPerVote value from the storage. MemoryPS's speed suffers a bit, but we don't use it for production environment. Part of #2322. Benchmark results: name old time/op new time/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 25.3µs ± 1% 26.4µs ± 9% +4.41% (p=0.043 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 27.9µs ± 1% 30.1µs ±15% +7.97% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 55.1µs ± 1% 60.2µs ± 7% +9.27% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 353µs ± 2% 416µs ±13% +17.88% (p=0.000 n=8+8) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 195µs ± 1% 216µs ± 7% +10.42% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 200µs ± 4% 214µs ± 9% +6.99% (p=0.002 n=9+8) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 223µs ± 2% 247µs ± 9% +10.60% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 612µs ±23% 855µs ±52% +39.60% (p=0.001 n=9+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 11.3ms ±53% 10.7ms ±50% ~ (p=0.739 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 12.0ms ±37% 10.4ms ±65% ~ (p=0.853 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 11.3ms ±40% 10.4ms ±49% ~ (p=0.631 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 3.80ms ±45% 3.69ms ±27% ~ (p=0.931 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 23.0µs ± 9% 22.6µs ± 4% ~ (p=0.059 n=8+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 25.9µs ± 5% 24.8µs ± 4% -4.17% (p=0.006 n=10+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 42.7µs ±13% 38.9µs ± 1% -8.85% (p=0.000 n=9+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 80.8µs ±12% 84.9µs ± 9% ~ (p=0.114 n=8+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 64.3µs ±16% 22.1µs ±23% -65.64% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 61.0µs ±34% 23.2µs ± 8% -62.04% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 62.2µs ±14% 25.7µs ±13% -58.66% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 359µs ±60% 325µs ±60% ~ (p=0.739 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 242µs ±21% 13µs ±28% -94.49% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 229µs ±23% 18µs ±70% -92.02% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 238µs ±28% 20µs ±109% -91.38% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 265µs ±20% 77µs ±62% -71.04% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 25.5µs ± 3% 24.7µs ± 7% ~ (p=0.143 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 27.4µs ± 2% 27.9µs ± 6% ~ (p=0.280 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 50.2µs ± 7% 47.4µs ±10% ~ (p=0.156 n=9+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 98.2µs ± 9% 94.6µs ±10% ~ (p=0.218 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 82.9µs ±13% 32.1µs ±22% -61.30% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 92.2µs ±11% 33.7µs ±12% -63.42% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 88.3µs ±22% 39.4µs ±14% -55.36% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 106µs ±18% 78µs ±24% -26.20% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 360µs ±24% 29µs ±53% -91.91% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 353µs ±16% 50µs ±70% -85.72% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 381µs ±20% 47µs ±111% -87.64% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 434µs ±19% 113µs ±41% -74.04% (p=0.000 n=10+10) name old alloc/op new alloc/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 4.82kB ± 0% 4.26kB ± 1% -11.62% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 4.99kB ± 0% 4.41kB ± 1% -11.56% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 8.45kB ± 0% 7.87kB ± 0% -6.88% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 55.0kB ± 0% 54.5kB ± 0% -0.81% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 29.1kB ± 0% 21.7kB ± 2% -25.56% (p=0.000 n=9+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 29.3kB ± 1% 21.8kB ± 2% -25.74% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 31.3kB ± 1% 23.6kB ± 1% -24.50% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 92.5kB ± 5% 84.7kB ± 3% -8.50% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 324kB ±29% 222kB ±44% -31.33% (p=0.007 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 308kB ±32% 174kB ±14% -43.56% (p=0.000 n=10+8) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 298kB ±23% 178kB ±36% -40.26% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 362kB ± 6% 248kB ± 6% -31.54% (p=0.004 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 5.15kB ± 3% 4.64kB ± 2% -9.92% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 5.36kB ± 1% 4.75kB ± 5% -11.42% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 8.15kB ± 4% 7.53kB ± 1% -7.62% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 33.2kB ± 5% 33.2kB ± 7% ~ (p=0.829 n=8+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 20.1kB ± 7% 5.8kB ±13% -70.90% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 19.8kB ±14% 6.2kB ± 5% -68.87% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 21.7kB ± 6% 8.0kB ± 7% -63.20% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 98.5kB ±44% 81.8kB ±48% ~ (p=0.143 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 130kB ± 4% 4kB ± 9% -96.69% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 131kB ± 4% 5kB ±21% -96.48% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 132kB ± 4% 6kB ±10% -95.39% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 151kB ± 4% 26kB ±10% -82.46% (p=0.000 n=9+9) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 5.92kB ± 3% 5.32kB ± 2% -10.01% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 6.09kB ± 2% 5.48kB ± 2% -10.00% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 9.61kB ± 1% 9.00kB ± 0% -6.29% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 33.4kB ± 7% 32.2kB ± 5% -3.60% (p=0.037 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 22.3kB ±10% 9.0kB ±16% -59.78% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 23.6kB ± 6% 8.5kB ±20% -63.76% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 24.2kB ± 9% 11.5kB ± 4% -52.34% (p=0.000 n=10+8) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 44.2kB ± 6% 30.8kB ± 9% -30.24% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 144kB ± 4% 10kB ±24% -93.39% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 146kB ± 1% 11kB ±37% -92.14% (p=0.000 n=7+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 149kB ± 3% 11kB ±12% -92.28% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 171kB ± 4% 34kB ±12% -80.00% (p=0.000 n=10+10) name old allocs/op new allocs/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 95.0 ± 0% 74.0 ± 0% -22.11% (p=0.001 n=8+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 100 ± 0% 78 ± 1% -21.70% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 153 ± 0% 131 ± 2% -14.25% (p=0.000 n=6+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 799 ± 2% 797 ± 4% ~ (p=0.956 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 438 ± 6% 167 ± 0% -61.86% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 446 ± 5% 172 ± 0% -61.38% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 506 ± 4% 232 ± 1% -54.21% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 1.31k ± 5% 0.97k ± 4% -26.20% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 5.06k ± 1% 1.09k ± 2% -78.53% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 5.02k ± 3% 1.08k ± 0% -78.45% (p=0.000 n=10+8) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 5.09k ± 3% 1.15k ± 2% -77.48% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 5.83k ± 1% 1.87k ± 3% -68.02% (p=0.004 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 103 ± 2% 82 ± 1% -20.83% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 107 ± 0% 86 ± 0% -19.63% (p=0.000 n=8+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 164 ± 1% 139 ± 0% -15.45% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 820 ± 1% 789 ± 1% -3.70% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 475 ± 0% 94 ± 3% -80.15% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 481 ± 0% 100 ± 2% -79.26% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 549 ± 0% 161 ± 2% -70.69% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 1.61k ±19% 1.19k ±25% -26.05% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 4.12k ± 0% 0.08k ± 2% -98.02% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 4.14k ± 0% 0.09k ± 3% -97.90% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 4.19k ± 0% 0.15k ± 3% -96.52% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 4.82k ± 1% 0.74k ± 1% -84.58% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 112 ± 4% 90 ± 3% -19.45% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 116 ± 2% 95 ± 2% -17.90% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 170 ± 3% 148 ± 3% -12.99% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 800 ± 2% 772 ± 2% -3.50% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 480 ± 3% 118 ± 3% -75.32% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 479 ± 2% 123 ± 3% -74.33% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 542 ± 1% 183 ± 3% -66.34% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 1.19k ± 1% 0.79k ± 1% -33.41% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 4.21k ± 1% 0.13k ±21% -96.83% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 4.23k ± 1% 0.15k ±17% -96.48% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 4.27k ± 0% 0.19k ± 6% -95.51% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 4.89k ± 1% 0.79k ± 2% -83.80% (p=0.000 n=10+10)
379 lines
11 KiB
Go
379 lines
11 KiB
Go
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"sort"
|
|
"testing"
|
|
|
|
"github.com/nspcc-dev/neo-go/internal/random"
|
|
"github.com/nspcc-dev/neo-go/pkg/util/slice"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func testMemCachedStorePersist(t *testing.T, ps Store) {
|
|
// cached Store
|
|
ts := NewMemCachedStore(ps)
|
|
// persisting nothing should do nothing
|
|
c, err := ts.Persist()
|
|
assert.Equal(t, nil, err)
|
|
assert.Equal(t, 0, c)
|
|
// persisting one key should result in one key in ps and nothing in ts
|
|
assert.NoError(t, ts.Put([]byte("key"), []byte("value")))
|
|
checkBatch(t, ts, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key"), Value: []byte("value")}}}, nil)
|
|
c, err = ts.Persist()
|
|
checkBatch(t, ts, nil, nil)
|
|
assert.Equal(t, nil, err)
|
|
assert.Equal(t, 1, c)
|
|
v, err := ps.Get([]byte("key"))
|
|
assert.Equal(t, nil, err)
|
|
assert.Equal(t, []byte("value"), v)
|
|
v, err = ts.MemoryStore.Get([]byte("key"))
|
|
assert.Equal(t, ErrKeyNotFound, err)
|
|
assert.Equal(t, []byte(nil), v)
|
|
// now we overwrite the previous `key` contents and also add `key2`,
|
|
assert.NoError(t, ts.Put([]byte("key"), []byte("newvalue")))
|
|
assert.NoError(t, ts.Put([]byte("key2"), []byte("value2")))
|
|
// this is to check that now key is written into the ps before we do
|
|
// persist
|
|
v, err = ps.Get([]byte("key2"))
|
|
assert.Equal(t, ErrKeyNotFound, err)
|
|
assert.Equal(t, []byte(nil), v)
|
|
checkBatch(t, ts, []KeyValueExists{
|
|
{KeyValue: KeyValue{Key: []byte("key"), Value: []byte("newvalue")}, Exists: true},
|
|
{KeyValue: KeyValue{Key: []byte("key2"), Value: []byte("value2")}},
|
|
}, nil)
|
|
// two keys should be persisted (one overwritten and one new) and
|
|
// available in the ps
|
|
c, err = ts.Persist()
|
|
checkBatch(t, ts, nil, nil)
|
|
assert.Equal(t, nil, err)
|
|
assert.Equal(t, 2, c)
|
|
v, err = ts.MemoryStore.Get([]byte("key"))
|
|
assert.Equal(t, ErrKeyNotFound, err)
|
|
assert.Equal(t, []byte(nil), v)
|
|
v, err = ts.MemoryStore.Get([]byte("key2"))
|
|
assert.Equal(t, ErrKeyNotFound, err)
|
|
assert.Equal(t, []byte(nil), v)
|
|
v, err = ps.Get([]byte("key"))
|
|
assert.Equal(t, nil, err)
|
|
assert.Equal(t, []byte("newvalue"), v)
|
|
v, err = ps.Get([]byte("key2"))
|
|
assert.Equal(t, nil, err)
|
|
assert.Equal(t, []byte("value2"), v)
|
|
checkBatch(t, ts, nil, nil)
|
|
// we've persisted some values, make sure successive persist is a no-op
|
|
c, err = ts.Persist()
|
|
assert.Equal(t, nil, err)
|
|
assert.Equal(t, 0, c)
|
|
// test persisting deletions
|
|
err = ts.Delete([]byte("key"))
|
|
assert.Equal(t, nil, err)
|
|
checkBatch(t, ts, nil, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key")}, Exists: true}})
|
|
c, err = ts.Persist()
|
|
checkBatch(t, ts, nil, nil)
|
|
assert.Equal(t, nil, err)
|
|
assert.Equal(t, 0, c)
|
|
v, err = ps.Get([]byte("key"))
|
|
assert.Equal(t, ErrKeyNotFound, err)
|
|
assert.Equal(t, []byte(nil), v)
|
|
v, err = ps.Get([]byte("key2"))
|
|
assert.Equal(t, nil, err)
|
|
assert.Equal(t, []byte("value2"), v)
|
|
}
|
|
|
|
func checkBatch(t *testing.T, ts *MemCachedStore, put []KeyValueExists, del []KeyValueExists) {
|
|
b := ts.GetBatch()
|
|
assert.Equal(t, len(put), len(b.Put), "wrong number of put elements in a batch")
|
|
assert.Equal(t, len(del), len(b.Deleted), "wrong number of deleted elements in a batch")
|
|
|
|
for i := range put {
|
|
assert.Contains(t, b.Put, put[i])
|
|
}
|
|
|
|
for i := range del {
|
|
assert.Contains(t, b.Deleted, del[i])
|
|
}
|
|
}
|
|
|
|
func TestMemCachedPersist(t *testing.T) {
|
|
t.Run("MemoryStore", func(t *testing.T) {
|
|
ps := NewMemoryStore()
|
|
testMemCachedStorePersist(t, ps)
|
|
})
|
|
t.Run("MemoryCachedStore", func(t *testing.T) {
|
|
ps1 := NewMemoryStore()
|
|
ps2 := NewMemCachedStore(ps1)
|
|
testMemCachedStorePersist(t, ps2)
|
|
})
|
|
t.Run("BoltDBStore", func(t *testing.T) {
|
|
ps := newBoltStoreForTesting(t)
|
|
t.Cleanup(func() {
|
|
err := ps.Close()
|
|
require.NoError(t, err)
|
|
})
|
|
testMemCachedStorePersist(t, ps)
|
|
})
|
|
}
|
|
|
|
func TestCachedGetFromPersistent(t *testing.T) {
|
|
key := []byte("key")
|
|
value := []byte("value")
|
|
ps := NewMemoryStore()
|
|
ts := NewMemCachedStore(ps)
|
|
|
|
assert.NoError(t, ps.Put(key, value))
|
|
val, err := ts.Get(key)
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, value, val)
|
|
assert.NoError(t, ts.Delete(key))
|
|
val, err = ts.Get(key)
|
|
assert.Equal(t, err, ErrKeyNotFound)
|
|
assert.Nil(t, val)
|
|
}
|
|
|
|
func TestCachedSeek(t *testing.T) {
|
|
var (
|
|
// Given this prefix...
|
|
goodPrefix = []byte{'f'}
|
|
// these pairs should be found...
|
|
lowerKVs = []KeyValue{
|
|
{[]byte("foo"), []byte("bar")},
|
|
{[]byte("faa"), []byte("bra")},
|
|
}
|
|
// and these should be not.
|
|
deletedKVs = []KeyValue{
|
|
{[]byte("fee"), []byte("pow")},
|
|
{[]byte("fii"), []byte("qaz")},
|
|
}
|
|
// and these should be not.
|
|
updatedKVs = []KeyValue{
|
|
{[]byte("fuu"), []byte("wop")},
|
|
{[]byte("fyy"), []byte("zaq")},
|
|
}
|
|
ps = NewMemoryStore()
|
|
ts = NewMemCachedStore(ps)
|
|
)
|
|
for _, v := range lowerKVs {
|
|
require.NoError(t, ps.Put(v.Key, v.Value))
|
|
}
|
|
for _, v := range deletedKVs {
|
|
require.NoError(t, ps.Put(v.Key, v.Value))
|
|
require.NoError(t, ts.Delete(v.Key))
|
|
}
|
|
for _, v := range updatedKVs {
|
|
require.NoError(t, ps.Put(v.Key, []byte("stub")))
|
|
require.NoError(t, ts.Put(v.Key, v.Value))
|
|
}
|
|
foundKVs := make(map[string][]byte)
|
|
ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool {
|
|
foundKVs[string(k)] = v
|
|
return true
|
|
})
|
|
assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs))
|
|
for _, kv := range lowerKVs {
|
|
assert.Equal(t, kv.Value, foundKVs[string(kv.Key)])
|
|
}
|
|
for _, kv := range deletedKVs {
|
|
_, ok := foundKVs[string(kv.Key)]
|
|
assert.Equal(t, false, ok)
|
|
}
|
|
for _, kv := range updatedKVs {
|
|
assert.Equal(t, kv.Value, foundKVs[string(kv.Key)])
|
|
}
|
|
}
|
|
|
|
func benchmarkCachedSeek(t *testing.B, ps Store, psElementsCount, tsElementsCount int) {
|
|
var (
|
|
searchPrefix = []byte{1}
|
|
badPrefix = []byte{2}
|
|
lowerPrefixGood = append(searchPrefix, 1)
|
|
lowerPrefixBad = append(badPrefix, 1)
|
|
deletedPrefixGood = append(searchPrefix, 2)
|
|
deletedPrefixBad = append(badPrefix, 2)
|
|
updatedPrefixGood = append(searchPrefix, 3)
|
|
updatedPrefixBad = append(badPrefix, 3)
|
|
|
|
ts = NewMemCachedStore(ps)
|
|
)
|
|
for i := 0; i < psElementsCount; i++ {
|
|
// lower KVs with matching prefix that should be found
|
|
require.NoError(t, ps.Put(append(lowerPrefixGood, random.Bytes(10)...), []byte("value")))
|
|
// lower KVs with non-matching prefix that shouldn't be found
|
|
require.NoError(t, ps.Put(append(lowerPrefixBad, random.Bytes(10)...), []byte("value")))
|
|
|
|
// deleted KVs with matching prefix that shouldn't be found
|
|
key := append(deletedPrefixGood, random.Bytes(10)...)
|
|
require.NoError(t, ps.Put(key, []byte("deleted")))
|
|
if i < tsElementsCount {
|
|
require.NoError(t, ts.Delete(key))
|
|
}
|
|
// deleted KVs with non-matching prefix that shouldn't be found
|
|
key = append(deletedPrefixBad, random.Bytes(10)...)
|
|
require.NoError(t, ps.Put(key, []byte("deleted")))
|
|
if i < tsElementsCount {
|
|
require.NoError(t, ts.Delete(key))
|
|
}
|
|
|
|
// updated KVs with matching prefix that should be found
|
|
key = append(updatedPrefixGood, random.Bytes(10)...)
|
|
require.NoError(t, ps.Put(key, []byte("stub")))
|
|
if i < tsElementsCount {
|
|
require.NoError(t, ts.Put(key, []byte("updated")))
|
|
}
|
|
// updated KVs with non-matching prefix that shouldn't be found
|
|
key = append(updatedPrefixBad, random.Bytes(10)...)
|
|
require.NoError(t, ps.Put(key, []byte("stub")))
|
|
if i < tsElementsCount {
|
|
require.NoError(t, ts.Put(key, []byte("updated")))
|
|
}
|
|
}
|
|
|
|
t.ReportAllocs()
|
|
t.ResetTimer()
|
|
for n := 0; n < t.N; n++ {
|
|
ts.Seek(SeekRange{Prefix: searchPrefix}, func(k, v []byte) bool { return true })
|
|
}
|
|
t.StopTimer()
|
|
}
|
|
|
|
func BenchmarkCachedSeek(t *testing.B) {
|
|
var stores = map[string]func(testing.TB) Store{
|
|
"MemPS": func(t testing.TB) Store {
|
|
return NewMemoryStore()
|
|
},
|
|
"BoltPS": newBoltStoreForTesting,
|
|
"LevelPS": newLevelDBForTesting,
|
|
}
|
|
for psName, newPS := range stores {
|
|
for psCount := 100; psCount <= 10000; psCount *= 10 {
|
|
for tsCount := 10; tsCount <= psCount; tsCount *= 10 {
|
|
t.Run(fmt.Sprintf("%s_%dTSItems_%dPSItems", psName, tsCount, psCount), func(t *testing.B) {
|
|
ps := newPS(t)
|
|
benchmarkCachedSeek(t, ps, psCount, tsCount)
|
|
ps.Close()
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func newMemCachedStoreForTesting(t testing.TB) Store {
|
|
return NewMemCachedStore(NewMemoryStore())
|
|
}
|
|
|
|
type BadBatch struct{}
|
|
|
|
func (b BadBatch) Delete(k []byte) {}
|
|
func (b BadBatch) Put(k, v []byte) {}
|
|
|
|
type BadStore struct {
|
|
onPutBatch func()
|
|
}
|
|
|
|
func (b *BadStore) Batch() Batch {
|
|
return BadBatch{}
|
|
}
|
|
func (b *BadStore) Delete(k []byte) error {
|
|
return nil
|
|
}
|
|
func (b *BadStore) Get([]byte) ([]byte, error) {
|
|
return nil, ErrKeyNotFound
|
|
}
|
|
func (b *BadStore) Put(k, v []byte) error {
|
|
return nil
|
|
}
|
|
func (b *BadStore) PutBatch(Batch) error {
|
|
return nil
|
|
}
|
|
func (b *BadStore) PutChangeSet(_ map[string][]byte, _ map[string]bool) error {
|
|
b.onPutBatch()
|
|
return ErrKeyNotFound
|
|
}
|
|
func (b *BadStore) Seek(rng SeekRange, f func(k, v []byte) bool) {
|
|
}
|
|
func (b *BadStore) Close() error {
|
|
return nil
|
|
}
|
|
|
|
func TestMemCachedPersistFailing(t *testing.T) {
|
|
var (
|
|
bs BadStore
|
|
t1 = []byte("t1")
|
|
t2 = []byte("t2")
|
|
b1 = []byte("b1")
|
|
)
|
|
// cached Store
|
|
ts := NewMemCachedStore(&bs)
|
|
// Set a pair of keys.
|
|
require.NoError(t, ts.Put(t1, t1))
|
|
require.NoError(t, ts.Put(t2, t2))
|
|
// This will be called during Persist().
|
|
bs.onPutBatch = func() {
|
|
// Drop one, add one.
|
|
require.NoError(t, ts.Put(b1, b1))
|
|
require.NoError(t, ts.Delete(t1))
|
|
}
|
|
_, err := ts.Persist()
|
|
require.Error(t, err)
|
|
// PutBatch() failed in Persist, but we still should have proper state.
|
|
_, err = ts.Get(t1)
|
|
require.Error(t, err)
|
|
res, err := ts.Get(t2)
|
|
require.NoError(t, err)
|
|
require.Equal(t, t2, res)
|
|
res, err = ts.Get(b1)
|
|
require.NoError(t, err)
|
|
require.Equal(t, b1, res)
|
|
}
|
|
|
|
func TestCachedSeekSorting(t *testing.T) {
|
|
var (
|
|
// Given this prefix...
|
|
goodPrefix = []byte{1}
|
|
// these pairs should be found...
|
|
lowerKVs = []KeyValue{
|
|
{[]byte{1, 2, 3}, []byte("bra")},
|
|
{[]byte{1, 2, 5}, []byte("bar")},
|
|
{[]byte{1, 3, 3}, []byte("bra")},
|
|
{[]byte{1, 3, 5}, []byte("bra")},
|
|
}
|
|
// and these should be not.
|
|
deletedKVs = []KeyValue{
|
|
{[]byte{1, 7, 3}, []byte("pow")},
|
|
{[]byte{1, 7, 4}, []byte("qaz")},
|
|
}
|
|
// and these should be not.
|
|
updatedKVs = []KeyValue{
|
|
{[]byte{1, 2, 4}, []byte("zaq")},
|
|
{[]byte{1, 2, 6}, []byte("zaq")},
|
|
{[]byte{1, 3, 2}, []byte("wop")},
|
|
{[]byte{1, 3, 4}, []byte("zaq")},
|
|
}
|
|
ps = NewMemoryStore()
|
|
ts = NewMemCachedStore(ps)
|
|
)
|
|
for _, v := range lowerKVs {
|
|
require.NoError(t, ps.Put(v.Key, v.Value))
|
|
}
|
|
for _, v := range deletedKVs {
|
|
require.NoError(t, ps.Put(v.Key, v.Value))
|
|
require.NoError(t, ts.Delete(v.Key))
|
|
}
|
|
for _, v := range updatedKVs {
|
|
require.NoError(t, ps.Put(v.Key, []byte("stub")))
|
|
require.NoError(t, ts.Put(v.Key, v.Value))
|
|
}
|
|
var foundKVs []KeyValue
|
|
ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool {
|
|
foundKVs = append(foundKVs, KeyValue{Key: slice.Copy(k), Value: slice.Copy(v)})
|
|
return true
|
|
})
|
|
assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs))
|
|
expected := append(lowerKVs, updatedKVs...)
|
|
sort.Slice(expected, func(i, j int) bool {
|
|
return bytes.Compare(expected[i].Key, expected[j].Key) < 0
|
|
})
|
|
require.Equal(t, expected, foundKVs)
|
|
}
|