Merge pull request #381 from nspcc-dev/add_boltd

storage: add boltdb support, closes #335.

config/protocol.mainnet.yml

@@ -30,7 +30,7 @@ ProtocolConfiguration:
 
 ApplicationConfiguration:
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/mainnet"
@@ -38,6 +38,8 @@ ApplicationConfiguration:
   #      Addr: "localhost:6379"
   #      Password: ""
   #      DB: 0
+  #    BoltDBOptions:
+  #      FilePath: "./chains/mainnet.bolt"
   RPCPort: 20332
   NodePort: 20333
   Relay: true

config/protocol.privnet.docker.four.yml

@@ -18,7 +18,7 @@ ProtocolConfiguration:
 
 ApplicationConfiguration:
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/privnet"
@@ -26,6 +26,8 @@ ApplicationConfiguration:
   #      Addr: "localhost:6379"
   #      Password: ""
   #      DB: 0
+  #    BoltDBOptions:
+  #      FilePath: "./chains/privnet.bolt"
   RPCPort: 20336
   NodePort: 20337
   Relay: true

config/protocol.privnet.docker.one.yml

@@ -15,7 +15,7 @@ ProtocolConfiguration:
 
 ApplicationConfiguration:
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/privnet"
@@ -23,6 +23,8 @@ ApplicationConfiguration:
   #      Addr: "localhost:6379"
   #      Password: ""
   #      DB: 0
+  #    BoltDBOptions:
+  #      FilePath: "./chains/privnet.bolt"
   RPCPort: 20333
   NodePort: 20334
   Relay: true

config/protocol.privnet.docker.three.yml

@@ -15,7 +15,7 @@ ProtocolConfiguration:
 
 ApplicationConfiguration:
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/privnet"
@@ -23,6 +23,8 @@ ApplicationConfiguration:
   #      Addr: "localhost:6379"
   #      Password: ""
   #      DB: 0
+  #    BoltDBOptions:
+  #      FilePath: "./chains/privnet.bolt"
   RPCPort: 20335
   NodePort: 20336
   Relay: true

config/protocol.privnet.docker.two.yml

@@ -15,7 +15,7 @@ ProtocolConfiguration:
 
 ApplicationConfiguration:
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/privnet"
@@ -23,6 +23,8 @@ ApplicationConfiguration:
   #      Addr: "localhost:6379"
   #      Password: ""
   #      DB: 0
+  #    BoltDBOptions:
+  #      FilePath: "./chains/privnet.bolt"
   RPCPort: 20334
   NodePort: 20335
   Relay: true

config/protocol.privnet.yml

@@ -21,7 +21,7 @@ ProtocolConfiguration:
 
 ApplicationConfiguration:
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/privnet"
@@ -29,6 +29,8 @@ ApplicationConfiguration:
   #      Addr: "localhost:6379"
   #      Password: ""
   #      DB: 0
+  #    BoltDBOptions:
+  #      FilePath: "./chains/privnet.bolt"
   RPCPort: 20331
   NodePort: 20332
   Relay: true

config/protocol.testnet.yml

@@ -30,7 +30,7 @@ ProtocolConfiguration:
 
 ApplicationConfiguration:
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','redis'.
+    Type: "leveldb" #other options: 'inmemory','redis','boltdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/testnet"
@@ -38,6 +38,8 @@ ApplicationConfiguration:
   #      Addr: "localhost:6379"
   #      Password: ""
   #      DB: 0
+  #    BoltDBOptions:
+  #      FilePath: "./chains/testnet.bolt"
   RPCPort: 20332
   NodePort: 20333
   Relay: true

config/protocol.unit_testnet.yml

@@ -20,7 +20,7 @@ ProtocolConfiguration:
 
 ApplicationConfiguration:
   DBConfiguration:
-    Type: "inmemory" #other options: 'inmemory','redis'.
+    Type: "inmemory" #other options: 'inmemory','redis','boltdb'.
     # DB type options. Uncomment those you need in case you want to switch DB type.
   #    LevelDBOptions:
   #        DataDirectoryPath: "./chains/unit_testnet"
@@ -28,6 +28,8 @@ ApplicationConfiguration:
   #      Addr: "localhost:6379"
   #      Password: ""
   #      DB: 0
+  #    BoltDBOptions:
+  #      FilePath: "./chains/unit_testnet.bolt"
   RPCPort: 20332
   NodePort: 20333
   Relay: true

go.mod

@@ -5,7 +5,7 @@ require (
 	github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db // indirect
 	github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect
 	github.com/alicebob/miniredis v2.5.0+incompatible
-	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/etcd-io/bbolt v1.3.3
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/go-redis/redis v6.10.2+incompatible
@@ -18,12 +18,13 @@ require (
 	github.com/nspcc-dev/rfc6979 v0.1.0
 	github.com/onsi/gomega v1.4.2 // indirect
 	github.com/pkg/errors v0.8.0
-	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.0.5
-	github.com/stretchr/testify v1.2.1
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.3.0
 	github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73
 	github.com/urfave/cli v1.20.0
 	github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 // indirect
+	go.etcd.io/bbolt v1.3.3 // indirect
 	golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb
 	golang.org/x/text v0.3.0
 	golang.org/x/tools v0.0.0-20180318012157-96caea41033d

go.sum

@@ -17,6 +17,10 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI=
 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=
+github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-redis/redis v6.10.2+incompatible h1:SLbqrO/Ik1nhXA5/cbEs1P5MUBo1Qq4ihlNfGnnipPw=
@@ -50,14 +54,21 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73 h1:I2drr5K0tykBofr74ZEGliE/Hf6fNkEbcPyFvsy7wZk=
 github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
 github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 h1:1b6PAtenNyhsmo/NKXVe34h7JEZKva1YB/ne7K7mqKM=
 github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb h1:O6ztCaemiMr99EgJdgXrr0J7N0EQN1oky/0GxML9Avk=
 golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=

pkg/core/storage/boltdb_store.go

@@ -0,0 +1,136 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/etcd-io/bbolt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// BoltDBOptions configuration for boltdb.
+type BoltDBOptions struct {
+	FilePath string `yaml:"FilePath"`
+}
+
+// Bucket represents bucket used in boltdb to store all the data.
+var Bucket = []byte("DB")
+
+// BoltDBStore it is the storage implementation for storing and retrieving
+// blockchain data.
+type BoltDBStore struct {
+	db *bbolt.DB
+}
+
+// BoltDBBatch simple batch implementation to satisfy the Store interface.
+type BoltDBBatch struct {
+	mem map[*[]byte][]byte
+}
+
+// Len implements the Batch interface.
+func (b *BoltDBBatch) Len() int {
+	return len(b.mem)
+}
+
+// Put implements the Batch interface.
+func (b *BoltDBBatch) Put(k, v []byte) {
+	b.mem[&k] = v
+}
+
+// NewBoltDBStore returns a new ready to use BoltDB storage with created bucket.
+func NewBoltDBStore(ctx context.Context, cfg BoltDBOptions) (*BoltDBStore, error) {
+	var opts *bbolt.Options       // should be exposed via BoltDBOptions if anything needed
+	fileMode := os.FileMode(0600) // should be exposed via BoltDBOptions if anything needed
+	fileName := cfg.FilePath
+	dir := path.Dir(fileName)
+	err := os.MkdirAll(dir, os.ModePerm)
+	if err != nil {
+		return nil, fmt.Errorf("could not create dir for BoltDB: %v", err)
+	}
+	db, err := bbolt.Open(fileName, fileMode, opts)
+	if err != nil {
+		return nil, err
+	}
+	err = db.Update(func(tx *bbolt.Tx) error {
+		_, err = tx.CreateBucketIfNotExists(Bucket)
+		if err != nil {
+			return fmt.Errorf("could not create root bucket: %v", err)
+		}
+		return nil
+	})
+
+	// graceful shutdown
+	go func() {
+		<-ctx.Done()
+		db.Close()
+	}()
+
+	return &BoltDBStore{db: db}, nil
+}
+
+// Put implements the Store interface.
+func (s *BoltDBStore) Put(key, value []byte) error {
+	return s.db.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(Bucket)
+		err := b.Put(key, value)
+		return err
+	})
+}
+
+// Get implements the Store interface.
+func (s *BoltDBStore) Get(key []byte) (val []byte, err error) {
+	err = s.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(Bucket)
+		val = b.Get(key)
+		return nil
+	})
+	if val == nil {
+		err = ErrKeyNotFound
+	}
+	return
+}
+
+// PutBatch implements the Store interface.
+func (s *BoltDBStore) PutBatch(batch Batch) error {
+	return s.db.Batch(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(Bucket)
+		for k, v := range batch.(*BoltDBBatch).mem {
+			err := b.Put(*k, v)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+// Seek implements the Store interface.
+func (s *BoltDBStore) Seek(key []byte, f func(k, v []byte)) {
+	err := s.db.View(func(tx *bbolt.Tx) error {
+		c := tx.Bucket(Bucket).Cursor()
+		prefix := util.BytesPrefix(key)
+		for k, v := c.Seek(prefix.Start); k != nil && bytes.Compare(k, prefix.Limit) <= 0; k, v = c.Next() {
+			f(k, v)
+		}
+		return nil
+	})
+	if err != nil {
+		fmt.Println("error while executing seek in boltDB")
+	}
+}
+
+// Batch implements the Batch interface and returns a boltdb
+// compatible Batch.
+func (s *BoltDBStore) Batch() Batch {
+	return &BoltDBBatch{
+		mem: make(map[*[]byte][]byte),
+	}
+}
+
+// Close releases all db resources.
+func (s *BoltDBStore) Close() error {
+	return s.db.Close()
+}

pkg/core/storage/boltdb_store_test.go

@@ -0,0 +1,89 @@
+package storage
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBoltDBBatch(t *testing.T) {
+	boltDB := BoltDBStore{}
+	want := &BoltDBBatch{mem: map[*[]byte][]byte{}}
+	if got := boltDB.Batch(); !reflect.DeepEqual(got, want) {
+		t.Errorf("BoltDB Batch() = %v, want %v", got, want)
+	}
+}
+
+func TestBoltDBBatch_Len(t *testing.T) {
+	batch := &BoltDBBatch{mem: map[*[]byte][]byte{}}
+	want := len(map[*[]byte][]byte{})
+	assert.Equal(t, want, batch.Len())
+}
+
+func TestBoltDBBatch_PutBatchAndGet(t *testing.T) {
+	key := []byte("foo")
+	value := []byte("bar")
+	batch := &BoltDBBatch{mem: map[*[]byte][]byte{&key: value}}
+
+	boltDBStore := openStore(t)
+
+	errPut := boltDBStore.PutBatch(batch)
+	assert.Nil(t, errPut, "Error while PutBatch")
+
+	result, err := boltDBStore.Get(key)
+	assert.Nil(t, err)
+	assert.Equal(t, value, result)
+
+	require.NoError(t, boltDBStore.Close())
+}
+
+func TestBoltDBBatch_PutAndGet(t *testing.T) {
+	key := []byte("foo")
+	value := []byte("bar")
+
+	boltDBStore := openStore(t)
+
+	errPut := boltDBStore.Put(key, value)
+	assert.Nil(t, errPut, "Error while Put")
+
+	result, err := boltDBStore.Get(key)
+	assert.Nil(t, err)
+	assert.Equal(t, value, result)
+
+	require.NoError(t, boltDBStore.Close())
+}
+
+func TestBoltDBStore_Seek(t *testing.T) {
+	key := []byte("foo")
+	value := []byte("bar")
+
+	boltDBStore := openStore(t)
+
+	errPut := boltDBStore.Put(key, value)
+	assert.Nil(t, errPut, "Error while Put")
+
+	boltDBStore.Seek(key, func(k, v []byte) {
+		assert.Equal(t, value, v)
+	})
+
+	require.NoError(t, boltDBStore.Close())
+}
+
+func openStore(t *testing.T) *BoltDBStore {
+	testFileName := "test_bolt_db"
+	file, err := ioutil.TempFile("", testFileName)
+	defer func() {
+		err := os.RemoveAll(testFileName)
+		require.NoError(t, err)
+	}()
+	require.NoError(t, err)
+	require.NoError(t, file.Close())
+	boltDBStore, err := NewBoltDBStore(context.Background(), BoltDBOptions{FilePath: testFileName})
+	require.NoError(t, err)
+	return boltDBStore
+}

pkg/core/storage/store.go

@@ -85,6 +85,8 @@ func NewStore(context context.Context, cfg DBConfiguration) (Store, error) {
 		store = NewMemoryStore()
 	case "redis":
 		store, err = NewRedisStore(cfg.RedisDBOptions)
+	case "boltdb":
+		store, err = NewBoltDBStore(context, cfg.BoltDBOptions)
 	}
 	return store, err
 }

pkg/core/storage/store_config.go

@@ -6,5 +6,6 @@ type (
 		Type           string         `yaml:"Type"`
 		LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"`
 		RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"`
+		BoltDBOptions  BoltDBOptions  `yaml:"BoltDBOptions"`
 	}
 )