From e111892653ea1a068733cba0ebb4bf64a551a31b Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 7 Oct 2019 19:45:01 +0300 Subject: [PATCH] storage: redo DB testing Make generic tests for all Store implementations, deduplicate tests. Implement Delete() tests, test Seek() better, add LevelDB tests (finally!). --- pkg/core/storage/boltdb_store_test.go | 58 +----- pkg/core/storage/leveldb_store_test.go | 40 +++++ pkg/core/storage/memory_store_test.go | 74 +------- pkg/core/storage/redis_store_test.go | 110 ++---------- pkg/core/storage/storeandbatch_test.go | 239 +++++++++++++++++++++++++ 5 files changed, 302 insertions(+), 219 deletions(-) create mode 100644 pkg/core/storage/storeandbatch_test.go diff --git a/pkg/core/storage/boltdb_store_test.go b/pkg/core/storage/boltdb_store_test.go index 299a74816..e9759b299 100644 --- a/pkg/core/storage/boltdb_store_test.go +++ b/pkg/core/storage/boltdb_store_test.go @@ -5,66 +5,10 @@ import ( "os" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestBoltDBBatch_PutBatchAndGet(t *testing.T) { - key := []byte("foo") - keycopy := make([]byte, len(key)) - copy(keycopy, key) - value := []byte("bar") - valuecopy := make([]byte, len(value)) - copy(valuecopy, value) - boltDBStore := openStore(t) - batch := boltDBStore.Batch() - - batch.Put(keycopy, valuecopy) - copy(valuecopy, key) - copy(keycopy, value) - errPut := boltDBStore.PutBatch(batch) - assert.Nil(t, errPut, "Error while PutBatch") - - result, err := boltDBStore.Get(key) - assert.Nil(t, err) - assert.Equal(t, value, result) - - require.NoError(t, boltDBStore.Close()) -} - -func TestBoltDBBatch_PutAndGet(t *testing.T) { - key := []byte("foo") - value := []byte("bar") - - boltDBStore := openStore(t) - - errPut := boltDBStore.Put(key, value) - assert.Nil(t, errPut, "Error while Put") - - result, err := boltDBStore.Get(key) - assert.Nil(t, err) - assert.Equal(t, value, result) - - require.NoError(t, boltDBStore.Close()) -} - -func TestBoltDBStore_Seek(t *testing.T) { - key := []byte("foo") - value := []byte("bar") - - boltDBStore := openStore(t) - - errPut := boltDBStore.Put(key, value) - assert.Nil(t, errPut, "Error while Put") - - boltDBStore.Seek(key, func(k, v []byte) { - assert.Equal(t, value, v) - }) - - require.NoError(t, boltDBStore.Close()) -} - -func openStore(t *testing.T) *BoltDBStore { +func newBoltStoreForTesting(t *testing.T) Store { testFileName := "test_bolt_db" file, err := ioutil.TempFile("", testFileName) defer func() { diff --git a/pkg/core/storage/leveldb_store_test.go b/pkg/core/storage/leveldb_store_test.go index 82be0547e..b8dcbbe83 100644 --- a/pkg/core/storage/leveldb_store_test.go +++ b/pkg/core/storage/leveldb_store_test.go @@ -1 +1,41 @@ package storage + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +type tempLevelDB struct { + LevelDBStore + dir string +} + +func (tldb *tempLevelDB) Close() error { + err := tldb.LevelDBStore.Close() + // Make test fail if failed to cleanup, even though technically it's + // not a LevelDBStore problem. + osErr := os.RemoveAll(tldb.dir) + if osErr != nil { + return osErr + } + return err +} + +func newLevelDBForTesting(t *testing.T) Store { + ldbDir, err := ioutil.TempDir(os.TempDir(), "testleveldb") + require.Nil(t, err, "failed to setup temporary directory") + + dbConfig := DBConfiguration{ + Type: "leveldb", + LevelDBOptions: LevelDBOptions{ + DataDirectoryPath: ldbDir, + }, + } + newLevelStore, err := NewLevelDBStore(dbConfig.LevelDBOptions) + require.Nil(t, err, "NewLevelDBStore error") + tldb := &tempLevelDB{LevelDBStore: *newLevelStore, dir: ldbDir} + return tldb +} diff --git a/pkg/core/storage/memory_store_test.go b/pkg/core/storage/memory_store_test.go index 96a7f5da5..1cf9f6d9a 100644 --- a/pkg/core/storage/memory_store_test.go +++ b/pkg/core/storage/memory_store_test.go @@ -4,78 +4,8 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -func TestGetPut(t *testing.T) { - var ( - s = NewMemoryStore() - key = []byte("sparse") - value = []byte("rocks") - ) - - if err := s.Put(key, value); err != nil { - t.Fatal(err) - } - - newVal, err := s.Get(key) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, value, newVal) - require.NoError(t, s.Close()) -} - -func TestKeyNotExist(t *testing.T) { - var ( - s = NewMemoryStore() - key = []byte("sparse") - ) - - _, err := s.Get(key) - assert.NotNil(t, err) - assert.Equal(t, err.Error(), "key not found") - require.NoError(t, s.Close()) -} - -func TestPutBatch(t *testing.T) { - var ( - s = NewMemoryStore() - key = []byte("sparse") - value = []byte("rocks") - batch = s.Batch() - ) - - batch.Put(key, value) - - if err := s.PutBatch(batch); err != nil { - t.Fatal(err) - } - - newVal, err := s.Get(key) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, value, newVal) - require.NoError(t, s.Close()) -} - -func TestMemoryStore_Seek(t *testing.T) { - var ( - s = NewMemoryStore() - key = []byte("sparse") - value = []byte("rocks") - ) - - if err := s.Put(key, value); err != nil { - t.Fatal(err) - } - - s.Seek(key, func(k, v []byte) { - assert.Equal(t, value, v) - }) -} - func TestMemoryStorePersist(t *testing.T) { // temporary Store ts := NewMemoryStore() @@ -138,3 +68,7 @@ func TestMemoryStorePersist(t *testing.T) { assert.Equal(t, nil, err) assert.Equal(t, []byte("value2"), v) } + +func newMemoryStoreForTesting(t *testing.T) Store { + return NewMemoryStore() +} diff --git a/pkg/core/storage/redis_store_test.go b/pkg/core/storage/redis_store_test.go index 919c5683f..5c82c506c 100644 --- a/pkg/core/storage/redis_store_test.go +++ b/pkg/core/storage/redis_store_test.go @@ -4,101 +4,18 @@ import ( "testing" "github.com/alicebob/miniredis" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestNewRedisStore(t *testing.T) { - redisMock, redisStore := prepareRedisMock(t) - key := []byte("testKey") - value := []byte("testValue") - err := redisStore.Put(key, value) - assert.Nil(t, err, "NewRedisStore Put error") - - result, err := redisStore.Get(key) - assert.Nil(t, err, "NewRedisStore Get error") - - assert.Equal(t, value, result) - require.NoError(t, redisStore.Close()) - redisMock.Close() -} - -func TestRedisStore_GetAndPut(t *testing.T) { - prepareRedisMock(t) - type args struct { - k []byte - v []byte - kToLook []byte - } - tests := []struct { - name string - args args - want []byte - wantErr bool - }{ - {"TestRedisStore_Get_Strings", - args{ - k: []byte("foo"), - v: []byte("bar"), - kToLook: []byte("foo"), - }, - []byte("bar"), - false, - }, - {"TestRedisStore_Get_Negative_Strings", - args{ - k: []byte("foo"), - v: []byte("bar"), - kToLook: []byte("wrong"), - }, - []byte(nil), - true, - }, - } - redisMock, redisStore := prepareRedisMock(t) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := redisStore.Put(tt.args.k, tt.args.v) - assert.Nil(t, err, "Got error while Put operation processing") - got, err := redisStore.Get(tt.args.kToLook) - if (err != nil) != tt.wantErr { - t.Errorf("Get() error = %v, wantErr %v", err, tt.wantErr) - return - } - assert.Equal(t, tt.want, got) - redisMock.FlushDB() - }) - } - require.NoError(t, redisStore.Close()) - redisMock.Close() -} - -func TestRedisStore_PutBatch(t *testing.T) { - batch := &MemoryBatch{m: map[string][]byte{"foo1": []byte("bar1")}} - mock, redisStore := prepareRedisMock(t) - err := redisStore.PutBatch(batch) - assert.Nil(t, err, "Error while PutBatch") - result, err := redisStore.Get([]byte("foo1")) - assert.Nil(t, err) - assert.Equal(t, []byte("bar1"), result) - require.NoError(t, redisStore.Close()) - mock.Close() -} - -func TestRedisStore_Seek(t *testing.T) { - mock, redisStore := prepareRedisMock(t) - redisStore.Seek([]byte("foo"), func(k, v []byte) { - assert.Equal(t, []byte("bar"), v) - }) - require.NoError(t, redisStore.Close()) - mock.Close() +type mockedRedisStore struct { + RedisStore + mini *miniredis.Miniredis } func prepareRedisMock(t *testing.T) (*miniredis.Miniredis, *RedisStore) { miniRedis, err := miniredis.Run() - if err != nil { - t.Errorf("MiniRedis mock creation error = %v", err) - } + require.Nil(t, err, "MiniRedis mock creation error") + _ = miniRedis.Set("foo", "bar") dbConfig := DBConfiguration{ @@ -110,9 +27,18 @@ func prepareRedisMock(t *testing.T) (*miniredis.Miniredis, *RedisStore) { }, } newRedisStore, err := NewRedisStore(dbConfig.RedisDBOptions) - if err != nil { - t.Errorf("NewRedisStore() error = %v", err) - return nil, nil - } + require.Nil(t, err, "NewRedisStore() error") return miniRedis, newRedisStore } + +func (mrs *mockedRedisStore) Close() error { + err := mrs.RedisStore.Close() + mrs.mini.Close() + return err +} + +func newRedisStoreForTesting(t *testing.T) Store { + mock, rs := prepareRedisMock(t) + mrs := &mockedRedisStore{RedisStore: *rs, mini: mock} + return mrs +} diff --git a/pkg/core/storage/storeandbatch_test.go b/pkg/core/storage/storeandbatch_test.go new file mode 100644 index 000000000..76ecbe754 --- /dev/null +++ b/pkg/core/storage/storeandbatch_test.go @@ -0,0 +1,239 @@ +package storage + +import ( + "reflect" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type dbSetup struct { + name string + create func(*testing.T) Store +} + +type dbTestFunction func(*testing.T, Store) + +func testStoreClose(t *testing.T, s Store) { + require.NoError(t, s.Close()) +} + +func testStorePutAndGet(t *testing.T, s Store) { + key := []byte("foo") + value := []byte("bar") + + require.NoError(t, s.Put(key, value)) + + result, err := s.Get(key) + assert.Nil(t, err) + require.Equal(t, value, result) + + require.NoError(t, s.Close()) +} + +func testStoreGetNonExistent(t *testing.T, s Store) { + key := []byte("sparse") + + _, err := s.Get(key) + assert.Equal(t, err, ErrKeyNotFound) + require.NoError(t, s.Close()) +} + +func testStorePutBatch(t *testing.T, s Store) { + var ( + key = []byte("foo") + value = []byte("bar") + batch = s.Batch() + ) + // Test that key and value are copied when batching. + keycopy := make([]byte, len(key)) + copy(keycopy, key) + valuecopy := make([]byte, len(value)) + copy(valuecopy, value) + + batch.Put(keycopy, valuecopy) + copy(valuecopy, key) + copy(keycopy, value) + + require.NoError(t, s.PutBatch(batch)) + newVal, err := s.Get(key) + assert.Nil(t, err) + require.Equal(t, value, newVal) + assert.Equal(t, value, newVal) + require.NoError(t, s.Close()) +} + +func testStoreSeek(t *testing.T, s Store) { + type kvSeen struct { + key []byte + val []byte + seen bool + } + var ( + // Given this prefix... + goodprefix = []byte{'f'} + // these pairs should be found... + goodkvs = []kvSeen{ + {[]byte("foo"), []byte("bar"), false}, + {[]byte("faa"), []byte("bra"), false}, + {[]byte("foox"), []byte("barx"), false}, + } + // and these should be not. + badkvs = []kvSeen{ + {[]byte("doo"), []byte("pow"), false}, + {[]byte("mew"), []byte("qaz"), false}, + } + ) + + for _, v := range goodkvs { + require.NoError(t, s.Put(v.key, v.val)) + } + for _, v := range badkvs { + require.NoError(t, s.Put(v.key, v.val)) + } + + numFound := 0 + s.Seek(goodprefix, func(k, v []byte) { + for i := 0; i < len(goodkvs); i++ { + if string(k) == string(goodkvs[i].key) { + assert.Equal(t, string(goodkvs[i].val), string(v)) + goodkvs[i].seen = true + } + } + for i := 0; i < len(badkvs); i++ { + if string(k) == string(badkvs[i].key) { + badkvs[i].seen = true + } + } + numFound++ + }) + assert.Equal(t, len(goodkvs), numFound) + for i := 0; i < len(goodkvs); i++ { + assert.Equal(t, true, goodkvs[i].seen) + } + for i := 0; i < len(badkvs); i++ { + assert.Equal(t, false, badkvs[i].seen) + } + require.NoError(t, s.Close()) +} + +func testStoreDeleteNonExistent(t *testing.T, s Store) { + key := []byte("sparse") + + assert.NoError(t, s.Delete(key)) + require.NoError(t, s.Close()) +} + +func testStorePutAndDelete(t *testing.T, s Store) { + key := []byte("foo") + value := []byte("bar") + + require.NoError(t, s.Put(key, value)) + + err := s.Delete(key) + assert.Nil(t, err) + + _, err = s.Get(key) + assert.NotNil(t, err) + assert.Equal(t, err, ErrKeyNotFound) + + // Double delete. + err = s.Delete(key) + assert.Nil(t, err) + + require.NoError(t, s.Close()) +} + +func testStorePutBatchWithDelete(t *testing.T, s Store) { + var ( + toBeStored = map[string][]byte{ + "foo": []byte("bar"), + "bar": []byte("baz"), + } + deletedInBatch = map[string][]byte{ + "edc": []byte("rfv"), + "tgb": []byte("yhn"), + } + readdedToBatch = map[string][]byte{ + "yhn": []byte("ujm"), + } + toBeDeleted = map[string][]byte{ + "qaz": []byte("wsx"), + "qwe": []byte("123"), + } + toStay = map[string][]byte{ + "key": []byte("val"), + "faa": []byte("bra"), + } + ) + for k, v := range toBeDeleted { + require.NoError(t, s.Put([]byte(k), v)) + } + for k, v := range toStay { + require.NoError(t, s.Put([]byte(k), v)) + } + batch := s.Batch() + for k, v := range toBeStored { + batch.Put([]byte(k), v) + } + for k := range toBeDeleted { + batch.Delete([]byte(k)) + } + for k, v := range readdedToBatch { + batch.Put([]byte(k), v) + } + for k, v := range deletedInBatch { + batch.Put([]byte(k), v) + } + for k := range deletedInBatch { + batch.Delete([]byte(k)) + } + for k := range readdedToBatch { + batch.Delete([]byte(k)) + } + for k, v := range readdedToBatch { + batch.Put([]byte(k), v) + } + require.NoError(t, s.PutBatch(batch)) + toBe := []map[string][]byte{toStay, toBeStored, readdedToBatch} + notToBe := []map[string][]byte{deletedInBatch, toBeDeleted} + for _, kvs := range toBe { + for k, v := range kvs { + value, err := s.Get([]byte(k)) + assert.Nil(t, err) + assert.Equal(t, value, v) + } + } + for _, kvs := range notToBe { + for k, v := range kvs { + _, err := s.Get([]byte(k)) + assert.Equal(t, ErrKeyNotFound, err, "%s:%s", k, v) + } + } + require.NoError(t, s.Close()) +} + +func TestAllDBs(t *testing.T) { + var DBs = []dbSetup{ + {"BoltDB", newBoltStoreForTesting}, + {"LevelDB", newLevelDBForTesting}, + {"Memory", newMemoryStoreForTesting}, + {"RedisDB", newRedisStoreForTesting}, + } + var tests = []dbTestFunction{testStoreClose, testStorePutAndGet, + testStoreGetNonExistent, testStorePutBatch, testStoreSeek, + testStoreDeleteNonExistent, testStorePutAndDelete, + testStorePutBatchWithDelete} + for _, db := range DBs { + for _, test := range tests { + s := db.create(t) + twrapper := func(t *testing.T) { + test(t, s) + } + fname := runtime.FuncForPC(reflect.ValueOf(test).Pointer()).Name() + t.Run(db.name+"/"+fname, twrapper) + } + } +}