From 54cdfe4a2373c029e994309a5093a1966a7c8584 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 8 Apr 2020 17:14:58 +0300 Subject: [PATCH] storage: add support of BadgerDB closes #820 --- config/protocol.mainnet.yml | 4 +- config/protocol.privnet.docker.four.yml | 4 +- config/protocol.privnet.docker.one.yml | 4 +- config/protocol.privnet.docker.single.yml | 4 +- config/protocol.privnet.docker.three.yml | 4 +- config/protocol.privnet.docker.two.yml | 4 +- config/protocol.privnet.yml | 4 +- config/protocol.testnet.yml | 4 +- config/protocol.unit_testnet.yml | 4 +- pkg/core/storage/badgerdb_store.go | 137 ++++++++++++++++++++++ pkg/core/storage/badgerdb_store_test.go | 44 +++++++ pkg/core/storage/store.go | 2 + pkg/core/storage/store_config.go | 11 +- pkg/core/storage/storeandbatch_test.go | 1 + 14 files changed, 217 insertions(+), 14 deletions(-) create mode 100644 pkg/core/storage/badgerdb_store.go create mode 100644 pkg/core/storage/badgerdb_store_test.go diff --git a/config/protocol.mainnet.yml b/config/protocol.mainnet.yml index 4cae5019c..80b944226 100644 --- a/config/protocol.mainnet.yml +++ b/config/protocol.mainnet.yml @@ -40,7 +40,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/mainnet" @@ -50,6 +50,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/mainnet.bolt" + # BadgerDBOptions: + # BadgerDir: "./chains/mainnet.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 10333 diff --git a/config/protocol.privnet.docker.four.yml b/config/protocol.privnet.docker.four.yml index 94e37b0ff..b21e89565 100644 --- a/config/protocol.privnet.docker.four.yml +++ b/config/protocol.privnet.docker.four.yml @@ -26,7 +26,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/four" @@ -36,6 +36,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # BadgerDBOptions: + # BadgerDir: "./chains/four.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20336 diff --git a/config/protocol.privnet.docker.one.yml b/config/protocol.privnet.docker.one.yml index 180db8c2b..464d3de0f 100644 --- a/config/protocol.privnet.docker.one.yml +++ b/config/protocol.privnet.docker.one.yml @@ -26,7 +26,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/one" @@ -36,6 +36,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # BadgerDBOptions: + # BadgerDir: "./chains/one.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20333 diff --git a/config/protocol.privnet.docker.single.yml b/config/protocol.privnet.docker.single.yml index 1a56ec1ad..c602a086e 100644 --- a/config/protocol.privnet.docker.single.yml +++ b/config/protocol.privnet.docker.single.yml @@ -20,7 +20,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/single" @@ -30,6 +30,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # BadgerDBOptions: + # BadgerDir: "./chains/single.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20333 diff --git a/config/protocol.privnet.docker.three.yml b/config/protocol.privnet.docker.three.yml index 4d3ab9553..7ad8665b6 100644 --- a/config/protocol.privnet.docker.three.yml +++ b/config/protocol.privnet.docker.three.yml @@ -26,7 +26,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/three" @@ -36,6 +36,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # BadgerDBOptions: + # BadgerDir: "./chains/three.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20335 diff --git a/config/protocol.privnet.docker.two.yml b/config/protocol.privnet.docker.two.yml index 1c6f80e02..4227379c3 100644 --- a/config/protocol.privnet.docker.two.yml +++ b/config/protocol.privnet.docker.two.yml @@ -26,7 +26,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/two" @@ -36,6 +36,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # BadgerDBOptions: + # BadgerDir: "./chains/two.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20334 diff --git a/config/protocol.privnet.yml b/config/protocol.privnet.yml index 839c47280..a396425c3 100644 --- a/config/protocol.privnet.yml +++ b/config/protocol.privnet.yml @@ -26,7 +26,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/privnet" @@ -36,6 +36,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # BadgerDBOptions: + # BadgerDir: "./chains/privnet.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20332 diff --git a/config/protocol.testnet.yml b/config/protocol.testnet.yml index 50d320b2b..711485d26 100644 --- a/config/protocol.testnet.yml +++ b/config/protocol.testnet.yml @@ -40,7 +40,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/testnet" @@ -50,6 +50,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/testnet.bolt" + # BadgerDBOptions: + # BadgerDir: "./chains/testnet.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20333 diff --git a/config/protocol.unit_testnet.yml b/config/protocol.unit_testnet.yml index 7a89c606f..f00fa93ff 100644 --- a/config/protocol.unit_testnet.yml +++ b/config/protocol.unit_testnet.yml @@ -25,7 +25,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "inmemory" #other options: 'inmemory','redis','boltdb'. + Type: "inmemory" #other options: 'inmemory','redis','boltdb', 'badgerdb'. # DB type options. Uncomment those you need in case you want to switch DB type. # LevelDBOptions: # DataDirectoryPath: "./chains/unit_testnet" @@ -35,6 +35,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/unit_testnet.bolt" + # BadgerDBOptions: + # BadgerDir: "./chains/unit_testnet.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20333 diff --git a/pkg/core/storage/badgerdb_store.go b/pkg/core/storage/badgerdb_store.go new file mode 100644 index 000000000..277487058 --- /dev/null +++ b/pkg/core/storage/badgerdb_store.go @@ -0,0 +1,137 @@ +package storage + +import ( + "os" + + "github.com/dgraph-io/badger/v2" +) + +// BadgerDBOptions configuration for BadgerDB. +type BadgerDBOptions struct { + Dir string `yaml:"BadgerDir"` +} + +// BadgerDBStore is the official storage implementation for storing and retrieving +// blockchain data. +type BadgerDBStore struct { + db *badger.DB +} + +// BadgerDBBatch is a wrapper around badger.WriteBatch, compatible with Batch interface +type BadgerDBBatch struct { + batch *badger.WriteBatch +} + +// Delete implements the Batch interface +func (b *BadgerDBBatch) Delete(key []byte) { + err := b.batch.Delete(key) + if err != nil { + panic(err) + } +} + +// Put implements the Batch interface +func (b *BadgerDBBatch) Put(key, value []byte) { + keycopy := make([]byte, len(key)) + copy(keycopy, key) + valuecopy := make([]byte, len(value)) + copy(valuecopy, value) + err := b.batch.Set(keycopy, valuecopy) + if err != nil { + panic(err) + } +} + +// NewBadgerDBStore returns a new BadgerDBStore object that will +// initialize the database found at the given path. +func NewBadgerDBStore(cfg BadgerDBOptions) (*BadgerDBStore, error) { + // BadgerDB isn't able to make nested directories + err := os.MkdirAll(cfg.Dir, os.ModePerm) + if err != nil { + panic(err) + } + opts := badger.DefaultOptions(cfg.Dir) // should be exposed via BadgerDBOptions if anything needed + + db, err := badger.Open(opts) + if err != nil { + return nil, err + } + + return &BadgerDBStore{ + db: db, + }, nil +} + +// Batch implements the Batch interface and returns a badgerdb +// compatible Batch. +func (b *BadgerDBStore) Batch() Batch { + return &BadgerDBBatch{b.db.NewWriteBatch()} +} + +// Delete implements the Store interface. +func (b *BadgerDBStore) Delete(key []byte) error { + return b.db.Update(func(txn *badger.Txn) error { + return txn.Delete(key) + }) +} + +// Get implements the Store interface. +func (b *BadgerDBStore) Get(key []byte) ([]byte, error) { + var val []byte + err := b.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(key) + if err == badger.ErrKeyNotFound { + return ErrKeyNotFound + } + val, err = item.ValueCopy(nil) + return err + }) + return val, err +} + +// Put implements the Store interface. +func (b *BadgerDBStore) Put(key, value []byte) error { + return b.db.Update(func(txn *badger.Txn) error { + err := txn.Set(key, value) + return err + }) +} + +// PutBatch implements the Store interface. +func (b *BadgerDBStore) PutBatch(batch Batch) error { + defer batch.(*BadgerDBBatch).batch.Cancel() + return batch.(*BadgerDBBatch).batch.Flush() +} + +// Seek implements the Store interface. +func (b *BadgerDBStore) Seek(key []byte, f func(k, v []byte)) { + err := b.db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.IteratorOptions{ + PrefetchValues: true, + PrefetchSize: 100, + Reverse: false, + AllVersions: false, + Prefix: key, + InternalAccess: false, + }) + defer it.Close() + for it.Seek(key); it.ValidForPrefix(key); it.Next() { + item := it.Item() + k := item.Key() + v, err := item.ValueCopy(nil) + if err != nil { + return err + } + f(k, v) + } + return nil + }) + if err != nil { + panic(err) + } +} + +// Close releases all db resources. +func (b *BadgerDBStore) Close() error { + return b.db.Close() +} diff --git a/pkg/core/storage/badgerdb_store_test.go b/pkg/core/storage/badgerdb_store_test.go new file mode 100644 index 000000000..820c716c8 --- /dev/null +++ b/pkg/core/storage/badgerdb_store_test.go @@ -0,0 +1,44 @@ +package storage + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +type tempBadgerDB struct { + *BadgerDBStore + dir string +} + +func (tbdb *tempBadgerDB) Close() error { + err := tbdb.BadgerDBStore.Close() + // Make test fail if failed to cleanup, even though technically it's + // not a BadgerDBStore problem. + osErr := os.RemoveAll(tbdb.dir) + if osErr != nil { + return osErr + } + return err +} + +func newBadgerDBForTesting(t *testing.T) Store { + bdbDir, err := ioutil.TempDir(os.TempDir(), "testbadgerdb") + require.Nil(t, err, "failed to setup temporary directory") + + dbConfig := DBConfiguration{ + Type: "badgerdb", + BadgerDBOptions: BadgerDBOptions{ + Dir: bdbDir, + }, + } + newBadgerStore, err := NewBadgerDBStore(dbConfig.BadgerDBOptions) + require.Nil(t, err, "NewBadgerDBStore error") + tbdb := &tempBadgerDB{ + BadgerDBStore: newBadgerStore, + dir: bdbDir, + } + return tbdb +} diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index bc4669701..73b3866b1 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -91,6 +91,8 @@ func NewStore(cfg DBConfiguration) (Store, error) { store, err = NewRedisStore(cfg.RedisDBOptions) case "boltdb": store, err = NewBoltDBStore(cfg.BoltDBOptions) + case "badgerdb": + store, err = NewBadgerDBStore(cfg.BadgerDBOptions) } return store, err } diff --git a/pkg/core/storage/store_config.go b/pkg/core/storage/store_config.go index 272008be4..34a65fc99 100644 --- a/pkg/core/storage/store_config.go +++ b/pkg/core/storage/store_config.go @@ -1,11 +1,12 @@ package storage type ( - // DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB'. + // DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB', 'boltDB', 'badgerDB'. DBConfiguration struct { - Type string `yaml:"Type"` - LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"` - RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"` - BoltDBOptions BoltDBOptions `yaml:"BoltDBOptions"` + Type string `yaml:"Type"` + LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"` + RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"` + BoltDBOptions BoltDBOptions `yaml:"BoltDBOptions"` + BadgerDBOptions BadgerDBOptions `yaml:"BadgerDBOptions"` } ) diff --git a/pkg/core/storage/storeandbatch_test.go b/pkg/core/storage/storeandbatch_test.go index c71f8cb1a..81fa37365 100644 --- a/pkg/core/storage/storeandbatch_test.go +++ b/pkg/core/storage/storeandbatch_test.go @@ -224,6 +224,7 @@ func TestAllDBs(t *testing.T) { {"MemCached", newMemCachedStoreForTesting}, {"Memory", newMemoryStoreForTesting}, {"RedisDB", newRedisStoreForTesting}, + {"BadgerDB", newBadgerDBForTesting}, } var tests = []dbTestFunction{testStoreClose, testStorePutAndGet, testStoreGetNonExistent, testStorePutBatch, testStoreSeek,