TrueCloudLab/neoneo-go
7
0
Fork 2
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #381 from nspcc-dev/add_boltd

Browse source 
storage: add boltdb support, closes #335.
This commit is contained in:
Roman Khimov 2019-09-14 13:00:52 +03:00 committed by GitHub
commit b21a220712
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 267 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
config/protocol.mainnet.yml
View file

@ -30,7 +30,7 @@ ProtocolConfiguration:
ApplicationConfiguration:
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/mainnet"
@ -38,6 +38,8 @@ ApplicationConfiguration:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/mainnet.bolt"
RPCPort: 20332
NodePort: 20333
Relay: true

4
config/protocol.privnet.docker.four.yml
View file

@ -18,7 +18,7 @@ ProtocolConfiguration:
ApplicationConfiguration:
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
@ -26,6 +26,8 @@ ApplicationConfiguration:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
RPCPort: 20336
NodePort: 20337
Relay: true

4
config/protocol.privnet.docker.one.yml
View file

@ -15,7 +15,7 @@ ProtocolConfiguration:
ApplicationConfiguration:
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
@ -23,6 +23,8 @@ ApplicationConfiguration:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
RPCPort: 20333
NodePort: 20334
Relay: true

4
config/protocol.privnet.docker.three.yml
View file

@ -15,7 +15,7 @@ ProtocolConfiguration:
ApplicationConfiguration:
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
@ -23,6 +23,8 @@ ApplicationConfiguration:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
RPCPort: 20335
NodePort: 20336
Relay: true

4
config/protocol.privnet.docker.two.yml
View file

@ -15,7 +15,7 @@ ProtocolConfiguration:
ApplicationConfiguration:
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
@ -23,6 +23,8 @@ ApplicationConfiguration:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
RPCPort: 20334
NodePort: 20335
Relay: true

4
config/protocol.privnet.yml
View file

@ -21,7 +21,7 @@ ProtocolConfiguration:
ApplicationConfiguration:
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
@ -29,6 +29,8 @@ ApplicationConfiguration:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/privnet.bolt"
RPCPort: 20331
NodePort: 20332
Relay: true

4
config/protocol.testnet.yml
View file

@ -30,7 +30,7 @@ ProtocolConfiguration:
ApplicationConfiguration:
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/testnet"
@ -38,6 +38,8 @@ ApplicationConfiguration:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/testnet.bolt"
RPCPort: 20332
NodePort: 20333
Relay: true

4
config/protocol.unit_testnet.yml
View file

@ -20,7 +20,7 @@ ProtocolConfiguration:
ApplicationConfiguration:
DBConfiguration:
Type: "inmemory" #other options: 'inmemory','redis'.
Type: "inmemory" #other options: 'inmemory','redis','boltdb'.
# DB type options. Uncomment those you need in case you want to switch DB type.
# LevelDBOptions:
# DataDirectoryPath: "./chains/unit_testnet"
@ -28,6 +28,8 @@ ApplicationConfiguration:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
# BoltDBOptions:
# FilePath: "./chains/unit_testnet.bolt"
RPCPort: 20332
NodePort: 20333
Relay: true

7
go.mod
View file

@ -5,7 +5,7 @@ require (
github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db // indirect
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect
github.com/alicebob/miniredis v2.5.0+incompatible
github.com/davecgh/go-spew v1.1.0 // indirect
github.com/etcd-io/bbolt v1.3.3
github.com/fatih/color v1.7.0 // indirect
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
github.com/go-redis/redis v6.10.2+incompatible
@ -18,12 +18,13 @@ require (
github.com/nspcc-dev/rfc6979 v0.1.0
github.com/onsi/gomega v1.4.2 // indirect
github.com/pkg/errors v0.8.0
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/sirupsen/logrus v1.0.5
github.com/stretchr/testify v1.2.1
github.com/stretchr/objx v0.2.0 // indirect
github.com/stretchr/testify v1.3.0
github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73
github.com/urfave/cli v1.20.0
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 // indirect
go.etcd.io/bbolt v1.3.3 // indirect
golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb
golang.org/x/text v0.3.0
golang.org/x/tools v0.0.0-20180318012157-96caea41033d

11
go.sum
View file

@ -17,6 +17,10 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI=
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-redis/redis v6.10.2+incompatible h1:SLbqrO/Ik1nhXA5/cbEs1P5MUBo1Qq4ihlNfGnnipPw=
@ -50,14 +54,21 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U=
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73 h1:I2drr5K0tykBofr74ZEGliE/Hf6fNkEbcPyFvsy7wZk=
github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 h1:1b6PAtenNyhsmo/NKXVe34h7JEZKva1YB/ne7K7mqKM=
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb h1:O6ztCaemiMr99EgJdgXrr0J7N0EQN1oky/0GxML9Avk=
golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=

136
pkg/core/storage/boltdb_store.go Normal file
View file

@ -0,0 +1,136 @@
package storage
import (
"bytes"
"context"
"fmt"
"os"
"path"
"github.com/etcd-io/bbolt"
"github.com/syndtr/goleveldb/leveldb/util"
)
// BoltDBOptions configuration for boltdb.
type BoltDBOptions struct {
FilePath string `yaml:"FilePath"`
}
// Bucket represents bucket used in boltdb to store all the data.
var Bucket = []byte("DB")
// BoltDBStore it is the storage implementation for storing and retrieving
// blockchain data.
type BoltDBStore struct {
db *bbolt.DB
}
// BoltDBBatch simple batch implementation to satisfy the Store interface.
type BoltDBBatch struct {
mem map[*[]byte][]byte
}
// Len implements the Batch interface.
func (b *BoltDBBatch) Len() int {
return len(b.mem)
}
// Put implements the Batch interface.
func (b *BoltDBBatch) Put(k, v []byte) {
b.mem[&k] = v
}
// NewBoltDBStore returns a new ready to use BoltDB storage with created bucket.
func NewBoltDBStore(ctx context.Context, cfg BoltDBOptions) (*BoltDBStore, error) {
var opts *bbolt.Options // should be exposed via BoltDBOptions if anything needed
fileMode := os.FileMode(0600) // should be exposed via BoltDBOptions if anything needed
fileName := cfg.FilePath
dir := path.Dir(fileName)
err := os.MkdirAll(dir, os.ModePerm)
if err != nil {
return nil, fmt.Errorf("could not create dir for BoltDB: %v", err)
}
db, err := bbolt.Open(fileName, fileMode, opts)
if err != nil {
return nil, err
}
err = db.Update(func(tx *bbolt.Tx) error {
_, err = tx.CreateBucketIfNotExists(Bucket)
if err != nil {
return fmt.Errorf("could not create root bucket: %v", err)
}
return nil
})
// graceful shutdown
go func() {
<-ctx.Done()
db.Close()
}()
return &BoltDBStore{db: db}, nil
}
// Put implements the Store interface.
func (s *BoltDBStore) Put(key, value []byte) error {
return s.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(Bucket)
err := b.Put(key, value)
return err
})
}
// Get implements the Store interface.
func (s *BoltDBStore) Get(key []byte) (val []byte, err error) {
err = s.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(Bucket)
val = b.Get(key)
return nil
})
if val == nil {
err = ErrKeyNotFound
}
return
}
// PutBatch implements the Store interface.
func (s *BoltDBStore) PutBatch(batch Batch) error {
return s.db.Batch(func(tx *bbolt.Tx) error {
b := tx.Bucket(Bucket)
for k, v := range batch.(*BoltDBBatch).mem {
err := b.Put(*k, v)
if err != nil {
return err
}
}
return nil
})
}
// Seek implements the Store interface.
func (s *BoltDBStore) Seek(key []byte, f func(k, v []byte)) {
err := s.db.View(func(tx *bbolt.Tx) error {
c := tx.Bucket(Bucket).Cursor()
prefix := util.BytesPrefix(key)
for k, v := c.Seek(prefix.Start); k != nil && bytes.Compare(k, prefix.Limit) <= 0; k, v = c.Next() {
f(k, v)
}
return nil
})
if err != nil {
fmt.Println("error while executing seek in boltDB")
}
}
// Batch implements the Batch interface and returns a boltdb
// compatible Batch.
func (s *BoltDBStore) Batch() Batch {
return &BoltDBBatch{
mem: make(map[*[]byte][]byte),
}
}
// Close releases all db resources.
func (s *BoltDBStore) Close() error {
return s.db.Close()
}

89
pkg/core/storage/boltdb_store_test.go Normal file
View file

@ -0,0 +1,89 @@
package storage
import (
"context"
"io/ioutil"
"os"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBoltDBBatch(t *testing.T) {
boltDB := BoltDBStore{}
want := &BoltDBBatch{mem: map[*[]byte][]byte{}}
if got := boltDB.Batch(); !reflect.DeepEqual(got, want) {
t.Errorf("BoltDB Batch() = %v, want %v", got, want)
}
}
func TestBoltDBBatch_Len(t *testing.T) {
batch := &BoltDBBatch{mem: map[*[]byte][]byte{}}
want := len(map[*[]byte][]byte{})
assert.Equal(t, want, batch.Len())
}
func TestBoltDBBatch_PutBatchAndGet(t *testing.T) {
key := []byte("foo")
value := []byte("bar")
batch := &BoltDBBatch{mem: map[*[]byte][]byte{&key: value}}
boltDBStore := openStore(t)
errPut := boltDBStore.PutBatch(batch)
assert.Nil(t, errPut, "Error while PutBatch")
result, err := boltDBStore.Get(key)
assert.Nil(t, err)
assert.Equal(t, value, result)
require.NoError(t, boltDBStore.Close())
}
func TestBoltDBBatch_PutAndGet(t *testing.T) {
key := []byte("foo")
value := []byte("bar")
boltDBStore := openStore(t)
errPut := boltDBStore.Put(key, value)
assert.Nil(t, errPut, "Error while Put")
result, err := boltDBStore.Get(key)
assert.Nil(t, err)
assert.Equal(t, value, result)
require.NoError(t, boltDBStore.Close())
}
func TestBoltDBStore_Seek(t *testing.T) {
key := []byte("foo")
value := []byte("bar")
boltDBStore := openStore(t)
errPut := boltDBStore.Put(key, value)
assert.Nil(t, errPut, "Error while Put")
boltDBStore.Seek(key, func(k, v []byte) {
assert.Equal(t, value, v)
})
require.NoError(t, boltDBStore.Close())
}
func openStore(t *testing.T) *BoltDBStore {
testFileName := "test_bolt_db"
file, err := ioutil.TempFile("", testFileName)
defer func() {
err := os.RemoveAll(testFileName)
require.NoError(t, err)
}()
require.NoError(t, err)
require.NoError(t, file.Close())
boltDBStore, err := NewBoltDBStore(context.Background(), BoltDBOptions{FilePath: testFileName})
require.NoError(t, err)
return boltDBStore
}

2
pkg/core/storage/store.go
View file

@ -85,6 +85,8 @@ func NewStore(context context.Context, cfg DBConfiguration) (Store, error) {
store = NewMemoryStore()
case "redis":
store, err = NewRedisStore(cfg.RedisDBOptions)
case "boltdb":
store, err = NewBoltDBStore(context, cfg.BoltDBOptions)
}
return store, err
}

1
pkg/core/storage/store_config.go
View file

@ -6,5 +6,6 @@ type (
Type string `yaml:"Type"`
LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"`
RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"`
BoltDBOptions BoltDBOptions `yaml:"BoltDBOptions"`
}
)