Merge pull request #839 from nspcc-dev/feature/badgerdb_support

storage: add support of BadgerDB
This commit is contained in:
Roman Khimov 2020-04-09 15:57:50 +03:00 committed by GitHub
commit bfaa025a34
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 217 additions and 14 deletions

View file

@ -40,7 +40,7 @@ ApplicationConfiguration:
# LogPath could be set up in case you need stdout logs to some proper file.
# LogPath: "./log/neogo.log"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/mainnet"
@ -50,6 +50,8 @@ ApplicationConfiguration:
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/mainnet.bolt"
# BadgerDBOptions:
# BadgerDir: "./chains/mainnet.badger"
# Uncomment in order to set up custom address for node.
# Address: 127.0.0.1
NodePort: 10333

View file

@ -26,7 +26,7 @@ ApplicationConfiguration:
# LogPath could be set up in case you need stdout logs to some proper file.
# LogPath: "./log/neogo.log"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "/chains/four"
@ -36,6 +36,8 @@ ApplicationConfiguration:
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
# BadgerDBOptions:
# BadgerDir: "./chains/four.badger"
# Uncomment in order to set up custom address for node.
# Address: 127.0.0.1
NodePort: 20336

View file

@ -26,7 +26,7 @@ ApplicationConfiguration:
# LogPath could be set up in case you need stdout logs to some proper file.
# LogPath: "./log/neogo.log"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "/chains/one"
@ -36,6 +36,8 @@ ApplicationConfiguration:
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
# BadgerDBOptions:
# BadgerDir: "./chains/one.badger"
# Uncomment in order to set up custom address for node.
# Address: 127.0.0.1
NodePort: 20333

View file

@ -20,7 +20,7 @@ ApplicationConfiguration:
# LogPath could be set up in case you need stdout logs to some proper file.
# LogPath: "./log/neogo.log"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "/chains/single"
@ -30,6 +30,8 @@ ApplicationConfiguration:
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
# BadgerDBOptions:
# BadgerDir: "./chains/single.badger"
# Uncomment in order to set up custom address for node.
# Address: 127.0.0.1
NodePort: 20333

View file

@ -26,7 +26,7 @@ ApplicationConfiguration:
# LogPath could be set up in case you need stdout logs to some proper file.
# LogPath: "./log/neogo.log"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "/chains/three"
@ -36,6 +36,8 @@ ApplicationConfiguration:
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
# BadgerDBOptions:
# BadgerDir: "./chains/three.badger"
# Uncomment in order to set up custom address for node.
# Address: 127.0.0.1
NodePort: 20335

View file

@ -26,7 +26,7 @@ ApplicationConfiguration:
# LogPath could be set up in case you need stdout logs to some proper file.
# LogPath: "./log/neogo.log"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "/chains/two"
@ -36,6 +36,8 @@ ApplicationConfiguration:
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
# BadgerDBOptions:
# BadgerDir: "./chains/two.badger"
# Uncomment in order to set up custom address for node.
# Address: 127.0.0.1
NodePort: 20334

View file

@ -26,7 +26,7 @@ ApplicationConfiguration:
# LogPath could be set up in case you need stdout logs to some proper file.
# LogPath: "./log/neogo.log"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
@ -36,6 +36,8 @@ ApplicationConfiguration:
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
# BadgerDBOptions:
# BadgerDir: "./chains/privnet.badger"
# Uncomment in order to set up custom address for node.
# Address: 127.0.0.1
NodePort: 20332

View file

@ -40,7 +40,7 @@ ApplicationConfiguration:
# LogPath could be set up in case you need stdout logs to some proper file.
# LogPath: "./log/neogo.log"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/testnet"
@ -50,6 +50,8 @@ ApplicationConfiguration:
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/testnet.bolt"
# BadgerDBOptions:
# BadgerDir: "./chains/testnet.badger"
# Uncomment in order to set up custom address for node.
# Address: 127.0.0.1
NodePort: 20333

View file

@ -25,7 +25,7 @@ ApplicationConfiguration:
# LogPath could be set up in case you need stdout logs to some proper file.
# LogPath: "./log/neogo.log"
DBConfiguration:
Type: "inmemory" #other options: 'inmemory','redis','boltdb'.
Type: "inmemory" #other options: 'inmemory','redis','boltdb', 'badgerdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
# LevelDBOptions:
# DataDirectoryPath: "./chains/unit_testnet"
@ -35,6 +35,8 @@ ApplicationConfiguration:
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/unit_testnet.bolt"
# BadgerDBOptions:
# BadgerDir: "./chains/unit_testnet.badger"
# Uncomment in order to set up custom address for node.
# Address: 127.0.0.1
NodePort: 20333

View file

@ -0,0 +1,137 @@
package storage
import (
"os"
"github.com/dgraph-io/badger/v2"
)
// BadgerDBOptions configuration for BadgerDB.
type BadgerDBOptions struct {
Dir string `yaml:"BadgerDir"`
}
// BadgerDBStore is the official storage implementation for storing and retrieving
// blockchain data.
type BadgerDBStore struct {
db *badger.DB
}
// BadgerDBBatch is a wrapper around badger.WriteBatch, compatible with Batch interface
type BadgerDBBatch struct {
batch *badger.WriteBatch
}
// Delete implements the Batch interface
func (b *BadgerDBBatch) Delete(key []byte) {
err := b.batch.Delete(key)
if err != nil {
panic(err)
}
}
// Put implements the Batch interface
func (b *BadgerDBBatch) Put(key, value []byte) {
keycopy := make([]byte, len(key))
copy(keycopy, key)
valuecopy := make([]byte, len(value))
copy(valuecopy, value)
err := b.batch.Set(keycopy, valuecopy)
if err != nil {
panic(err)
}
}
// NewBadgerDBStore returns a new BadgerDBStore object that will
// initialize the database found at the given path.
func NewBadgerDBStore(cfg BadgerDBOptions) (*BadgerDBStore, error) {
// BadgerDB isn't able to make nested directories
err := os.MkdirAll(cfg.Dir, os.ModePerm)
if err != nil {
panic(err)
}
opts := badger.DefaultOptions(cfg.Dir) // should be exposed via BadgerDBOptions if anything needed
db, err := badger.Open(opts)
if err != nil {
return nil, err
}
return &BadgerDBStore{
db: db,
}, nil
}
// Batch implements the Batch interface and returns a badgerdb
// compatible Batch.
func (b *BadgerDBStore) Batch() Batch {
return &BadgerDBBatch{b.db.NewWriteBatch()}
}
// Delete implements the Store interface.
func (b *BadgerDBStore) Delete(key []byte) error {
return b.db.Update(func(txn *badger.Txn) error {
return txn.Delete(key)
})
}
// Get implements the Store interface.
func (b *BadgerDBStore) Get(key []byte) ([]byte, error) {
var val []byte
err := b.db.View(func(txn *badger.Txn) error {
item, err := txn.Get(key)
if err == badger.ErrKeyNotFound {
return ErrKeyNotFound
}
val, err = item.ValueCopy(nil)
return err
})
return val, err
}
// Put implements the Store interface.
func (b *BadgerDBStore) Put(key, value []byte) error {
return b.db.Update(func(txn *badger.Txn) error {
err := txn.Set(key, value)
return err
})
}
// PutBatch implements the Store interface.
func (b *BadgerDBStore) PutBatch(batch Batch) error {
defer batch.(*BadgerDBBatch).batch.Cancel()
return batch.(*BadgerDBBatch).batch.Flush()
}
// Seek implements the Store interface.
func (b *BadgerDBStore) Seek(key []byte, f func(k, v []byte)) {
err := b.db.View(func(txn *badger.Txn) error {
it := txn.NewIterator(badger.IteratorOptions{
PrefetchValues: true,
PrefetchSize: 100,
Reverse: false,
AllVersions: false,
Prefix: key,
InternalAccess: false,
})
defer it.Close()
for it.Seek(key); it.ValidForPrefix(key); it.Next() {
item := it.Item()
k := item.Key()
v, err := item.ValueCopy(nil)
if err != nil {
return err
}
f(k, v)
}
return nil
})
if err != nil {
panic(err)
}
}
// Close releases all db resources.
func (b *BadgerDBStore) Close() error {
return b.db.Close()
}

View file

@ -0,0 +1,44 @@
package storage
import (
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/require"
)
type tempBadgerDB struct {
*BadgerDBStore
dir string
}
func (tbdb *tempBadgerDB) Close() error {
err := tbdb.BadgerDBStore.Close()
// Make test fail if failed to cleanup, even though technically it's
// not a BadgerDBStore problem.
osErr := os.RemoveAll(tbdb.dir)
if osErr != nil {
return osErr
}
return err
}
func newBadgerDBForTesting(t *testing.T) Store {
bdbDir, err := ioutil.TempDir(os.TempDir(), "testbadgerdb")
require.Nil(t, err, "failed to setup temporary directory")
dbConfig := DBConfiguration{
Type: "badgerdb",
BadgerDBOptions: BadgerDBOptions{
Dir: bdbDir,
},
}
newBadgerStore, err := NewBadgerDBStore(dbConfig.BadgerDBOptions)
require.Nil(t, err, "NewBadgerDBStore error")
tbdb := &tempBadgerDB{
BadgerDBStore: newBadgerStore,
dir: bdbDir,
}
return tbdb
}

View file

@ -91,6 +91,8 @@ func NewStore(cfg DBConfiguration) (Store, error) {
store, err = NewRedisStore(cfg.RedisDBOptions)
case "boltdb":
store, err = NewBoltDBStore(cfg.BoltDBOptions)
case "badgerdb":
store, err = NewBadgerDBStore(cfg.BadgerDBOptions)
}
return store, err
}

View file

@ -1,11 +1,12 @@
package storage
type (
// DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB'.
// DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB', 'boltDB', 'badgerDB'.
DBConfiguration struct {
Type string `yaml:"Type"`
LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"`
RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"`
BoltDBOptions BoltDBOptions `yaml:"BoltDBOptions"`
Type string `yaml:"Type"`
LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"`
RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"`
BoltDBOptions BoltDBOptions `yaml:"BoltDBOptions"`
BadgerDBOptions BadgerDBOptions `yaml:"BadgerDBOptions"`
}
)

View file

@ -224,6 +224,7 @@ func TestAllDBs(t *testing.T) {
{"MemCached", newMemCachedStoreForTesting},
{"Memory", newMemoryStoreForTesting},
{"RedisDB", newRedisStoreForTesting},
{"BadgerDB", newBadgerDBForTesting},
}
var tests = []dbTestFunction{testStoreClose, testStorePutAndGet,
testStoreGetNonExistent, testStorePutBatch, testStoreSeek,