forked from TrueCloudLab/neoneo-go
cd42b8b20c
This simple approach allows to improve the performance of BoltDB and LevelDB in both terms of speed and allocations for retrieving GasPerVote value from the storage. MemoryPS's speed suffers a bit, but we don't use it for production environment. Part of #2322. Benchmark results: name old time/op new time/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 25.3µs ± 1% 26.4µs ± 9% +4.41% (p=0.043 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 27.9µs ± 1% 30.1µs ±15% +7.97% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 55.1µs ± 1% 60.2µs ± 7% +9.27% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 353µs ± 2% 416µs ±13% +17.88% (p=0.000 n=8+8) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 195µs ± 1% 216µs ± 7% +10.42% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 200µs ± 4% 214µs ± 9% +6.99% (p=0.002 n=9+8) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 223µs ± 2% 247µs ± 9% +10.60% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 612µs ±23% 855µs ±52% +39.60% (p=0.001 n=9+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 11.3ms ±53% 10.7ms ±50% ~ (p=0.739 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 12.0ms ±37% 10.4ms ±65% ~ (p=0.853 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 11.3ms ±40% 10.4ms ±49% ~ (p=0.631 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 3.80ms ±45% 3.69ms ±27% ~ (p=0.931 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 23.0µs ± 9% 22.6µs ± 4% ~ (p=0.059 n=8+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 25.9µs ± 5% 24.8µs ± 4% -4.17% (p=0.006 n=10+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 42.7µs ±13% 38.9µs ± 1% -8.85% (p=0.000 n=9+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 80.8µs ±12% 84.9µs ± 9% ~ (p=0.114 n=8+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 64.3µs ±16% 22.1µs ±23% -65.64% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 61.0µs ±34% 23.2µs ± 8% -62.04% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 62.2µs ±14% 25.7µs ±13% -58.66% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 359µs ±60% 325µs ±60% ~ (p=0.739 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 242µs ±21% 13µs ±28% -94.49% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 229µs ±23% 18µs ±70% -92.02% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 238µs ±28% 20µs ±109% -91.38% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 265µs ±20% 77µs ±62% -71.04% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 25.5µs ± 3% 24.7µs ± 7% ~ (p=0.143 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 27.4µs ± 2% 27.9µs ± 6% ~ (p=0.280 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 50.2µs ± 7% 47.4µs ±10% ~ (p=0.156 n=9+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 98.2µs ± 9% 94.6µs ±10% ~ (p=0.218 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 82.9µs ±13% 32.1µs ±22% -61.30% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 92.2µs ±11% 33.7µs ±12% -63.42% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 88.3µs ±22% 39.4µs ±14% -55.36% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 106µs ±18% 78µs ±24% -26.20% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 360µs ±24% 29µs ±53% -91.91% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 353µs ±16% 50µs ±70% -85.72% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 381µs ±20% 47µs ±111% -87.64% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 434µs ±19% 113µs ±41% -74.04% (p=0.000 n=10+10) name old alloc/op new alloc/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 4.82kB ± 0% 4.26kB ± 1% -11.62% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 4.99kB ± 0% 4.41kB ± 1% -11.56% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 8.45kB ± 0% 7.87kB ± 0% -6.88% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 55.0kB ± 0% 54.5kB ± 0% -0.81% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 29.1kB ± 0% 21.7kB ± 2% -25.56% (p=0.000 n=9+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 29.3kB ± 1% 21.8kB ± 2% -25.74% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 31.3kB ± 1% 23.6kB ± 1% -24.50% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 92.5kB ± 5% 84.7kB ± 3% -8.50% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 324kB ±29% 222kB ±44% -31.33% (p=0.007 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 308kB ±32% 174kB ±14% -43.56% (p=0.000 n=10+8) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 298kB ±23% 178kB ±36% -40.26% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 362kB ± 6% 248kB ± 6% -31.54% (p=0.004 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 5.15kB ± 3% 4.64kB ± 2% -9.92% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 5.36kB ± 1% 4.75kB ± 5% -11.42% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 8.15kB ± 4% 7.53kB ± 1% -7.62% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 33.2kB ± 5% 33.2kB ± 7% ~ (p=0.829 n=8+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 20.1kB ± 7% 5.8kB ±13% -70.90% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 19.8kB ±14% 6.2kB ± 5% -68.87% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 21.7kB ± 6% 8.0kB ± 7% -63.20% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 98.5kB ±44% 81.8kB ±48% ~ (p=0.143 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 130kB ± 4% 4kB ± 9% -96.69% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 131kB ± 4% 5kB ±21% -96.48% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 132kB ± 4% 6kB ±10% -95.39% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 151kB ± 4% 26kB ±10% -82.46% (p=0.000 n=9+9) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 5.92kB ± 3% 5.32kB ± 2% -10.01% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 6.09kB ± 2% 5.48kB ± 2% -10.00% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 9.61kB ± 1% 9.00kB ± 0% -6.29% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 33.4kB ± 7% 32.2kB ± 5% -3.60% (p=0.037 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 22.3kB ±10% 9.0kB ±16% -59.78% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 23.6kB ± 6% 8.5kB ±20% -63.76% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 24.2kB ± 9% 11.5kB ± 4% -52.34% (p=0.000 n=10+8) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 44.2kB ± 6% 30.8kB ± 9% -30.24% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 144kB ± 4% 10kB ±24% -93.39% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 146kB ± 1% 11kB ±37% -92.14% (p=0.000 n=7+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 149kB ± 3% 11kB ±12% -92.28% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 171kB ± 4% 34kB ±12% -80.00% (p=0.000 n=10+10) name old allocs/op new allocs/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 95.0 ± 0% 74.0 ± 0% -22.11% (p=0.001 n=8+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 100 ± 0% 78 ± 1% -21.70% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 153 ± 0% 131 ± 2% -14.25% (p=0.000 n=6+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 799 ± 2% 797 ± 4% ~ (p=0.956 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 438 ± 6% 167 ± 0% -61.86% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 446 ± 5% 172 ± 0% -61.38% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 506 ± 4% 232 ± 1% -54.21% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 1.31k ± 5% 0.97k ± 4% -26.20% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 5.06k ± 1% 1.09k ± 2% -78.53% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 5.02k ± 3% 1.08k ± 0% -78.45% (p=0.000 n=10+8) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 5.09k ± 3% 1.15k ± 2% -77.48% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 5.83k ± 1% 1.87k ± 3% -68.02% (p=0.004 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 103 ± 2% 82 ± 1% -20.83% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 107 ± 0% 86 ± 0% -19.63% (p=0.000 n=8+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 164 ± 1% 139 ± 0% -15.45% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 820 ± 1% 789 ± 1% -3.70% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 475 ± 0% 94 ± 3% -80.15% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 481 ± 0% 100 ± 2% -79.26% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 549 ± 0% 161 ± 2% -70.69% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 1.61k ±19% 1.19k ±25% -26.05% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 4.12k ± 0% 0.08k ± 2% -98.02% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 4.14k ± 0% 0.09k ± 3% -97.90% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 4.19k ± 0% 0.15k ± 3% -96.52% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 4.82k ± 1% 0.74k ± 1% -84.58% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 112 ± 4% 90 ± 3% -19.45% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 116 ± 2% 95 ± 2% -17.90% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 170 ± 3% 148 ± 3% -12.99% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 800 ± 2% 772 ± 2% -3.50% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 480 ± 3% 118 ± 3% -75.32% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 479 ± 2% 123 ± 3% -74.33% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 542 ± 1% 183 ± 3% -66.34% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 1.19k ± 1% 0.79k ± 1% -33.41% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 4.21k ± 1% 0.13k ±21% -96.83% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 4.23k ± 1% 0.15k ±17% -96.48% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 4.27k ± 0% 0.19k ± 6% -95.51% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 4.89k ± 1% 0.79k ± 2% -83.80% (p=0.000 n=10+10)
326 lines
7.8 KiB
Go
326 lines
7.8 KiB
Go
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/nspcc-dev/neo-go/pkg/util/slice"
|
|
)
|
|
|
|
// MemCachedStore is a wrapper around persistent store that caches all changes
|
|
// being made for them to be later flushed in one batch.
|
|
type MemCachedStore struct {
|
|
MemoryStore
|
|
|
|
// plock protects Persist from double entrance.
|
|
plock sync.Mutex
|
|
// Persistent Store.
|
|
ps Store
|
|
}
|
|
|
|
type (
|
|
// KeyValue represents key-value pair.
|
|
KeyValue struct {
|
|
Key []byte
|
|
Value []byte
|
|
}
|
|
|
|
// KeyValueExists represents key-value pair with indicator whether the item
|
|
// exists in the persistent storage.
|
|
KeyValueExists struct {
|
|
KeyValue
|
|
|
|
Exists bool
|
|
}
|
|
|
|
// MemBatch represents a changeset to be persisted.
|
|
MemBatch struct {
|
|
Put []KeyValueExists
|
|
Deleted []KeyValueExists
|
|
}
|
|
)
|
|
|
|
// NewMemCachedStore creates a new MemCachedStore object.
|
|
func NewMemCachedStore(lower Store) *MemCachedStore {
|
|
return &MemCachedStore{
|
|
MemoryStore: *NewMemoryStore(),
|
|
ps: lower,
|
|
}
|
|
}
|
|
|
|
// Get implements the Store interface.
|
|
func (s *MemCachedStore) Get(key []byte) ([]byte, error) {
|
|
s.mut.RLock()
|
|
defer s.mut.RUnlock()
|
|
k := string(key)
|
|
if val, ok := s.mem[k]; ok {
|
|
return val, nil
|
|
}
|
|
if _, ok := s.del[k]; ok {
|
|
return nil, ErrKeyNotFound
|
|
}
|
|
return s.ps.Get(key)
|
|
}
|
|
|
|
// GetBatch returns currently accumulated changeset.
|
|
func (s *MemCachedStore) GetBatch() *MemBatch {
|
|
s.mut.RLock()
|
|
defer s.mut.RUnlock()
|
|
|
|
var b MemBatch
|
|
|
|
b.Put = make([]KeyValueExists, 0, len(s.mem))
|
|
for k, v := range s.mem {
|
|
key := []byte(k)
|
|
_, err := s.ps.Get(key)
|
|
b.Put = append(b.Put, KeyValueExists{KeyValue: KeyValue{Key: key, Value: v}, Exists: err == nil})
|
|
}
|
|
|
|
b.Deleted = make([]KeyValueExists, 0, len(s.del))
|
|
for k := range s.del {
|
|
key := []byte(k)
|
|
_, err := s.ps.Get(key)
|
|
b.Deleted = append(b.Deleted, KeyValueExists{KeyValue: KeyValue{Key: key}, Exists: err == nil})
|
|
}
|
|
|
|
return &b
|
|
}
|
|
|
|
// Seek implements the Store interface.
|
|
func (s *MemCachedStore) Seek(rng SeekRange, f func(k, v []byte) bool) {
|
|
s.seek(context.Background(), rng, false, f)
|
|
}
|
|
|
|
// SeekAsync returns non-buffered channel with matching KeyValue pairs. Key and
|
|
// value slices may not be copied and may be modified. SeekAsync can guarantee
|
|
// that key-value items are sorted by key in ascending way.
|
|
func (s *MemCachedStore) SeekAsync(ctx context.Context, rng SeekRange, cutPrefix bool) chan KeyValue {
|
|
res := make(chan KeyValue)
|
|
go func() {
|
|
s.seek(ctx, rng, cutPrefix, func(k, v []byte) bool {
|
|
res <- KeyValue{
|
|
Key: k,
|
|
Value: v,
|
|
}
|
|
return true // always continue, we have context for early stop.
|
|
})
|
|
close(res)
|
|
}()
|
|
|
|
return res
|
|
}
|
|
|
|
// seek is internal representations of Seek* capable of seeking for the given key
|
|
// and supporting early stop using provided context. `cutPrefix` denotes whether provided
|
|
// key needs to be cut off the resulting keys. `rng` specifies prefix items must match
|
|
// and point to start seeking from. Backwards seeking from some point is supported
|
|
// with corresponding `rng` field set.
|
|
func (s *MemCachedStore) seek(ctx context.Context, rng SeekRange, cutPrefix bool, f func(k, v []byte) bool) {
|
|
// Create memory store `mem` and `del` snapshot not to hold the lock.
|
|
var memRes []KeyValueExists
|
|
sPrefix := string(rng.Prefix)
|
|
lPrefix := len(sPrefix)
|
|
sStart := string(rng.Start)
|
|
lStart := len(sStart)
|
|
isKeyOK := func(key string) bool {
|
|
return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) >= 0)
|
|
}
|
|
if rng.Backwards {
|
|
isKeyOK = func(key string) bool {
|
|
return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) <= 0)
|
|
}
|
|
}
|
|
s.mut.RLock()
|
|
for k, v := range s.MemoryStore.mem {
|
|
if isKeyOK(k) {
|
|
memRes = append(memRes, KeyValueExists{
|
|
KeyValue: KeyValue{
|
|
Key: []byte(k),
|
|
Value: v,
|
|
},
|
|
Exists: true,
|
|
})
|
|
}
|
|
}
|
|
for k := range s.MemoryStore.del {
|
|
if isKeyOK(k) {
|
|
memRes = append(memRes, KeyValueExists{
|
|
KeyValue: KeyValue{
|
|
Key: []byte(k),
|
|
},
|
|
})
|
|
}
|
|
}
|
|
ps := s.ps
|
|
s.mut.RUnlock()
|
|
|
|
less := func(k1, k2 []byte) bool {
|
|
res := bytes.Compare(k1, k2)
|
|
return res != 0 && rng.Backwards == (res > 0)
|
|
}
|
|
// Sort memRes items for further comparison with ps items.
|
|
sort.Slice(memRes, func(i, j int) bool {
|
|
return less(memRes[i].Key, memRes[j].Key)
|
|
})
|
|
|
|
var (
|
|
done bool
|
|
iMem int
|
|
kvMem KeyValueExists
|
|
haveMem bool
|
|
)
|
|
if iMem < len(memRes) {
|
|
kvMem = memRes[iMem]
|
|
haveMem = true
|
|
iMem++
|
|
}
|
|
// Merge results of seek operations in ascending order. It returns whether iterating
|
|
// should be continued.
|
|
mergeFunc := func(k, v []byte) bool {
|
|
if done {
|
|
return false
|
|
}
|
|
kvPs := KeyValue{
|
|
Key: slice.Copy(k),
|
|
Value: slice.Copy(v),
|
|
}
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
done = true
|
|
return false
|
|
default:
|
|
var isMem = haveMem && less(kvMem.Key, kvPs.Key)
|
|
if isMem {
|
|
if kvMem.Exists {
|
|
if cutPrefix {
|
|
kvMem.Key = kvMem.Key[lPrefix:]
|
|
}
|
|
if !f(kvMem.Key, kvMem.Value) {
|
|
done = true
|
|
return false
|
|
}
|
|
}
|
|
if iMem < len(memRes) {
|
|
kvMem = memRes[iMem]
|
|
haveMem = true
|
|
iMem++
|
|
} else {
|
|
haveMem = false
|
|
}
|
|
} else {
|
|
if !bytes.Equal(kvMem.Key, kvPs.Key) {
|
|
if cutPrefix {
|
|
kvPs.Key = kvPs.Key[lPrefix:]
|
|
}
|
|
if !f(kvPs.Key, kvPs.Value) {
|
|
done = true
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
ps.Seek(rng, mergeFunc)
|
|
|
|
if !done && haveMem {
|
|
loop:
|
|
for i := iMem - 1; i < len(memRes); i++ {
|
|
select {
|
|
case <-ctx.Done():
|
|
break loop
|
|
default:
|
|
kvMem = memRes[i]
|
|
if kvMem.Exists {
|
|
if cutPrefix {
|
|
kvMem.Key = kvMem.Key[lPrefix:]
|
|
}
|
|
if !f(kvMem.Key, kvMem.Value) {
|
|
break loop
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Persist flushes all the MemoryStore contents into the (supposedly) persistent
|
|
// store ps. MemCachedStore remains accessible for the most part of this action
|
|
// (any new changes will be cached in memory).
|
|
func (s *MemCachedStore) Persist() (int, error) {
|
|
return s.persist(false)
|
|
}
|
|
|
|
// PersistSync flushes all the MemoryStore contents into the (supposedly) persistent
|
|
// store ps. It's different from Persist in that it blocks MemCachedStore completely
|
|
// while flushing things from memory to persistent store.
|
|
func (s *MemCachedStore) PersistSync() (int, error) {
|
|
return s.persist(true)
|
|
}
|
|
|
|
func (s *MemCachedStore) persist(isSync bool) (int, error) {
|
|
var err error
|
|
var keys, dkeys int
|
|
|
|
s.plock.Lock()
|
|
defer s.plock.Unlock()
|
|
s.mut.Lock()
|
|
|
|
keys = len(s.mem)
|
|
dkeys = len(s.del)
|
|
if keys == 0 && dkeys == 0 {
|
|
s.mut.Unlock()
|
|
return 0, nil
|
|
}
|
|
|
|
// tempstore technically copies current s in lower layer while real s
|
|
// starts using fresh new maps. This tempstore is only known here and
|
|
// nothing ever changes it, therefore accesses to it (reads) can go
|
|
// unprotected while writes are handled by s proper.
|
|
var tempstore = &MemCachedStore{MemoryStore: MemoryStore{mem: s.mem, del: s.del}, ps: s.ps}
|
|
s.ps = tempstore
|
|
s.mem = make(map[string][]byte, len(s.mem))
|
|
s.del = make(map[string]bool, len(s.del))
|
|
if !isSync {
|
|
s.mut.Unlock()
|
|
}
|
|
|
|
err = tempstore.ps.PutChangeSet(tempstore.mem, tempstore.del)
|
|
|
|
if !isSync {
|
|
s.mut.Lock()
|
|
}
|
|
if err == nil {
|
|
// tempstore.mem and tempstore.del are completely flushed now
|
|
// to tempstore.ps, so all KV pairs are the same and this
|
|
// substitution has no visible effects.
|
|
s.ps = tempstore.ps
|
|
} else {
|
|
// We're toast. We'll try to still keep proper state, but OOM
|
|
// killer will get to us eventually.
|
|
for k := range s.mem {
|
|
tempstore.put(k, s.mem[k])
|
|
}
|
|
for k := range s.del {
|
|
tempstore.drop(k)
|
|
}
|
|
s.ps = tempstore.ps
|
|
s.mem = tempstore.mem
|
|
s.del = tempstore.del
|
|
}
|
|
s.mut.Unlock()
|
|
return keys, err
|
|
}
|
|
|
|
// Close implements Store interface, clears up memory and closes the lower layer
|
|
// Store.
|
|
func (s *MemCachedStore) Close() error {
|
|
// It's always successful.
|
|
_ = s.MemoryStore.Close()
|
|
return s.ps.Close()
|
|
}
|