Merge pull request #839 from nspcc-dev/feature/badgerdb_support

storage: add support of BadgerDB

config/protocol.mainnet.yml

@@ -40,7 +40,7 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/mainnet"
@@ -50,6 +50,8 @@ ApplicationConfiguration:
   #      DB: 0
   #    BoltDBOptions:
   #      FilePath: "./chains/mainnet.bolt"
+  #    BadgerDBOptions:
+  #      BadgerDir: "./chains/mainnet.badger"
   #  Uncomment in order to set up custom address for node.
   #  Address: 127.0.0.1
   NodePort: 10333

config/protocol.privnet.docker.four.yml

@@ -26,7 +26,7 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "/chains/four"
@@ -36,6 +36,8 @@ ApplicationConfiguration:
   #      DB: 0
   #    BoltDBOptions:
   #      FilePath: "./chains/privnet.bolt"
+  #    BadgerDBOptions:
+  #      BadgerDir: "./chains/four.badger"
   #  Uncomment in order to set up custom address for node.
   #  Address: 127.0.0.1
   NodePort: 20336

config/protocol.privnet.docker.one.yml

@@ -26,7 +26,7 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "/chains/one"
@@ -36,6 +36,8 @@ ApplicationConfiguration:
   #      DB: 0
   #    BoltDBOptions:
   #      FilePath: "./chains/privnet.bolt"
+  #    BadgerDBOptions:
+  #      BadgerDir: "./chains/one.badger"
   #  Uncomment in order to set up custom address for node.
   #  Address: 127.0.0.1
   NodePort: 20333

config/protocol.privnet.docker.single.yml

@@ -20,7 +20,7 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "/chains/single"
@@ -30,6 +30,8 @@ ApplicationConfiguration:
   #      DB: 0
   #    BoltDBOptions:
   #      FilePath: "./chains/privnet.bolt"
+  #    BadgerDBOptions:
+  #      BadgerDir: "./chains/single.badger"
   #  Uncomment in order to set up custom address for node.
   #  Address: 127.0.0.1
   NodePort: 20333

config/protocol.privnet.docker.three.yml

@@ -26,7 +26,7 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "/chains/three"
@@ -36,6 +36,8 @@ ApplicationConfiguration:
   #      DB: 0
   #    BoltDBOptions:
   #      FilePath: "./chains/privnet.bolt"
+  #    BadgerDBOptions:
+  #      BadgerDir: "./chains/three.badger"
   #  Uncomment in order to set up custom address for node.
   #  Address: 127.0.0.1
   NodePort: 20335

config/protocol.privnet.docker.two.yml

@@ -26,7 +26,7 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "/chains/two"
@@ -36,6 +36,8 @@ ApplicationConfiguration:
   #      DB: 0
   #    BoltDBOptions:
   #      FilePath: "./chains/privnet.bolt"
+  #    BadgerDBOptions:
+  #      BadgerDir: "./chains/two.badger"
   #  Uncomment in order to set up custom address for node.
   #  Address: 127.0.0.1
   NodePort: 20334

config/protocol.privnet.yml

@@ -26,7 +26,7 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/privnet"
@@ -36,6 +36,8 @@ ApplicationConfiguration:
   #      DB: 0
   #    BoltDBOptions:
   #      FilePath: "./chains/privnet.bolt"
+  #  BadgerDBOptions:
+  #    BadgerDir: "./chains/privnet.badger"
   #  Uncomment in order to set up custom address for node.
   #  Address: 127.0.0.1
   NodePort: 20332

config/protocol.testnet.yml

@@ -40,7 +40,7 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/testnet"
@@ -50,6 +50,8 @@ ApplicationConfiguration:
   #      DB: 0
   #    BoltDBOptions:
   #      FilePath: "./chains/testnet.bolt"
+  #    BadgerDBOptions:
+  #      BadgerDir: "./chains/testnet.badger"
   #  Uncomment in order to set up custom address for node.
   #  Address: 127.0.0.1
   NodePort: 20333

config/protocol.unit_testnet.yml

@@ -25,7 +25,7 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "inmemory" #other options: 'inmemory','redis','boltdb'.
+    Type: "inmemory" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
   #    LevelDBOptions:
   #        DataDirectoryPath: "./chains/unit_testnet"
@@ -35,6 +35,8 @@ ApplicationConfiguration:
   #      DB: 0
   #    BoltDBOptions:
   #      FilePath: "./chains/unit_testnet.bolt"
+  #    BadgerDBOptions:
+  #      BadgerDir: "./chains/unit_testnet.badger"
   #  Uncomment in order to set up custom address for node.
   #  Address: 127.0.0.1
   NodePort: 20333

pkg/core/storage/badgerdb_store.go

@@ -0,0 +1,137 @@
+package storage
+
+import (
+	"os"
+
+	"github.com/dgraph-io/badger/v2"
+)
+
+// BadgerDBOptions configuration for BadgerDB.
+type BadgerDBOptions struct {
+	Dir string `yaml:"BadgerDir"`
+}
+
+// BadgerDBStore is the official storage implementation for storing and retrieving
+// blockchain data.
+type BadgerDBStore struct {
+	db *badger.DB
+}
+
+// BadgerDBBatch is a wrapper around badger.WriteBatch, compatible with Batch interface
+type BadgerDBBatch struct {
+	batch *badger.WriteBatch
+}
+
+// Delete implements the Batch interface
+func (b *BadgerDBBatch) Delete(key []byte) {
+	err := b.batch.Delete(key)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// Put implements the Batch interface
+func (b *BadgerDBBatch) Put(key, value []byte) {
+	keycopy := make([]byte, len(key))
+	copy(keycopy, key)
+	valuecopy := make([]byte, len(value))
+	copy(valuecopy, value)
+	err := b.batch.Set(keycopy, valuecopy)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// NewBadgerDBStore returns a new BadgerDBStore object that will
+// initialize the database found at the given path.
+func NewBadgerDBStore(cfg BadgerDBOptions) (*BadgerDBStore, error) {
+	// BadgerDB isn't able to make nested directories
+	err := os.MkdirAll(cfg.Dir, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	opts := badger.DefaultOptions(cfg.Dir) // should be exposed via BadgerDBOptions if anything needed
+
+	db, err := badger.Open(opts)
+	if err != nil {
+		return nil, err
+	}
+
+	return &BadgerDBStore{
+		db: db,
+	}, nil
+}
+
+// Batch implements the Batch interface and returns a badgerdb
+// compatible Batch.
+func (b *BadgerDBStore) Batch() Batch {
+	return &BadgerDBBatch{b.db.NewWriteBatch()}
+}
+
+// Delete implements the Store interface.
+func (b *BadgerDBStore) Delete(key []byte) error {
+	return b.db.Update(func(txn *badger.Txn) error {
+		return txn.Delete(key)
+	})
+}
+
+// Get implements the Store interface.
+func (b *BadgerDBStore) Get(key []byte) ([]byte, error) {
+	var val []byte
+	err := b.db.View(func(txn *badger.Txn) error {
+		item, err := txn.Get(key)
+		if err == badger.ErrKeyNotFound {
+			return ErrKeyNotFound
+		}
+		val, err = item.ValueCopy(nil)
+		return err
+	})
+	return val, err
+}
+
+// Put implements the Store interface.
+func (b *BadgerDBStore) Put(key, value []byte) error {
+	return b.db.Update(func(txn *badger.Txn) error {
+		err := txn.Set(key, value)
+		return err
+	})
+}
+
+// PutBatch implements the Store interface.
+func (b *BadgerDBStore) PutBatch(batch Batch) error {
+	defer batch.(*BadgerDBBatch).batch.Cancel()
+	return batch.(*BadgerDBBatch).batch.Flush()
+}
+
+// Seek implements the Store interface.
+func (b *BadgerDBStore) Seek(key []byte, f func(k, v []byte)) {
+	err := b.db.View(func(txn *badger.Txn) error {
+		it := txn.NewIterator(badger.IteratorOptions{
+			PrefetchValues: true,
+			PrefetchSize:   100,
+			Reverse:        false,
+			AllVersions:    false,
+			Prefix:         key,
+			InternalAccess: false,
+		})
+		defer it.Close()
+		for it.Seek(key); it.ValidForPrefix(key); it.Next() {
+			item := it.Item()
+			k := item.Key()
+			v, err := item.ValueCopy(nil)
+			if err != nil {
+				return err
+			}
+			f(k, v)
+		}
+		return nil
+	})
+	if err != nil {
+		panic(err)
+	}
+}
+
+// Close releases all db resources.
+func (b *BadgerDBStore) Close() error {
+	return b.db.Close()
+}

pkg/core/storage/badgerdb_store_test.go

@@ -0,0 +1,44 @@
+package storage
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type tempBadgerDB struct {
+	*BadgerDBStore
+	dir string
+}
+
+func (tbdb *tempBadgerDB) Close() error {
+	err := tbdb.BadgerDBStore.Close()
+	// Make test fail if failed to cleanup, even though technically it's
+	// not a BadgerDBStore problem.
+	osErr := os.RemoveAll(tbdb.dir)
+	if osErr != nil {
+		return osErr
+	}
+	return err
+}
+
+func newBadgerDBForTesting(t *testing.T) Store {
+	bdbDir, err := ioutil.TempDir(os.TempDir(), "testbadgerdb")
+	require.Nil(t, err, "failed to setup temporary directory")
+
+	dbConfig := DBConfiguration{
+		Type: "badgerdb",
+		BadgerDBOptions: BadgerDBOptions{
+			Dir: bdbDir,
+		},
+	}
+	newBadgerStore, err := NewBadgerDBStore(dbConfig.BadgerDBOptions)
+	require.Nil(t, err, "NewBadgerDBStore error")
+	tbdb := &tempBadgerDB{
+		BadgerDBStore: newBadgerStore,
+		dir:           bdbDir,
+	}
+	return tbdb
+}

pkg/core/storage/store.go

@@ -91,6 +91,8 @@ func NewStore(cfg DBConfiguration) (Store, error) {
 		store, err = NewRedisStore(cfg.RedisDBOptions)
 	case "boltdb":
 		store, err = NewBoltDBStore(cfg.BoltDBOptions)
+	case "badgerdb":
+		store, err = NewBadgerDBStore(cfg.BadgerDBOptions)
 	}
 	return store, err
 }

pkg/core/storage/store_config.go

@@ -1,11 +1,12 @@
 package storage
 
 type (
-	// DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB'.
+	// DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB', 'boltDB', 'badgerDB'.
 	DBConfiguration struct {
-		Type           string         `yaml:"Type"`
+		Type            string          `yaml:"Type"`
-		LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"`
+		LevelDBOptions  LevelDBOptions  `yaml:"LevelDBOptions"`
-		RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"`
+		RedisDBOptions  RedisDBOptions  `yaml:"RedisDBOptions"`
-		BoltDBOptions  BoltDBOptions  `yaml:"BoltDBOptions"`
+		BoltDBOptions   BoltDBOptions   `yaml:"BoltDBOptions"`
+		BadgerDBOptions BadgerDBOptions `yaml:"BadgerDBOptions"`
 	}
 )

pkg/core/storage/storeandbatch_test.go

@@ -224,6 +224,7 @@ func TestAllDBs(t *testing.T) {
 		{"MemCached", newMemCachedStoreForTesting},
 		{"Memory", newMemoryStoreForTesting},
 		{"RedisDB", newRedisStoreForTesting},
+		{"BadgerDB", newBadgerDBForTesting},
 	}
 	var tests = []dbTestFunction{testStoreClose, testStorePutAndGet,
 		testStoreGetNonExistent, testStorePutBatch, testStoreSeek,