From bfe5fd538ec76607ea397e28c7a073b2a12d78e7 Mon Sep 17 00:00:00 2001 From: Vsevolod Brekelov Date: Sat, 14 Sep 2019 10:28:43 +0300 Subject: [PATCH 1/2] storage: add boltDB support closes #335 --- config/protocol.mainnet.yml | 4 +- config/protocol.privnet.docker.four.yml | 4 +- config/protocol.privnet.docker.one.yml | 4 +- config/protocol.privnet.docker.three.yml | 4 +- config/protocol.privnet.docker.two.yml | 4 +- config/protocol.privnet.yml | 4 +- config/protocol.testnet.yml | 4 +- config/protocol.unit_testnet.yml | 4 +- go.mod | 7 +- go.sum | 11 ++ pkg/core/storage/boltdb_store.go | 136 +++++++++++++++++++++++ pkg/core/storage/store.go | 2 + pkg/core/storage/store_config.go | 1 + 13 files changed, 178 insertions(+), 11 deletions(-) create mode 100644 pkg/core/storage/boltdb_store.go diff --git a/config/protocol.mainnet.yml b/config/protocol.mainnet.yml index 776ba7ae3..6292ffe58 100644 --- a/config/protocol.mainnet.yml +++ b/config/protocol.mainnet.yml @@ -30,7 +30,7 @@ ProtocolConfiguration: ApplicationConfiguration: DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/mainnet" @@ -38,6 +38,8 @@ ApplicationConfiguration: # Addr: "localhost:6379" # Password: "" # DB: 0 + # BoltDBOptions: + # FilePath: "./chains/mainnet.bolt" RPCPort: 20332 NodePort: 20333 Relay: true diff --git a/config/protocol.privnet.docker.four.yml b/config/protocol.privnet.docker.four.yml index 9fbdc1430..d530183ff 100644 --- a/config/protocol.privnet.docker.four.yml +++ b/config/protocol.privnet.docker.four.yml @@ -18,7 +18,7 @@ ProtocolConfiguration: ApplicationConfiguration: DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/privnet" @@ -26,6 +26,8 @@ ApplicationConfiguration: # Addr: "localhost:6379" # Password: "" # DB: 0 + # BoltDBOptions: + # FilePath: "./chains/privnet.bolt" RPCPort: 20336 NodePort: 20337 Relay: true diff --git a/config/protocol.privnet.docker.one.yml b/config/protocol.privnet.docker.one.yml index 2ac00797a..96e88f07a 100644 --- a/config/protocol.privnet.docker.one.yml +++ b/config/protocol.privnet.docker.one.yml @@ -15,7 +15,7 @@ ProtocolConfiguration: ApplicationConfiguration: DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/privnet" @@ -23,6 +23,8 @@ ApplicationConfiguration: # Addr: "localhost:6379" # Password: "" # DB: 0 + # BoltDBOptions: + # FilePath: "./chains/privnet.bolt" RPCPort: 20333 NodePort: 20334 Relay: true diff --git a/config/protocol.privnet.docker.three.yml b/config/protocol.privnet.docker.three.yml index c2dbffbbc..82206637c 100644 --- a/config/protocol.privnet.docker.three.yml +++ b/config/protocol.privnet.docker.three.yml @@ -15,7 +15,7 @@ ProtocolConfiguration: ApplicationConfiguration: DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/privnet" @@ -23,6 +23,8 @@ ApplicationConfiguration: # Addr: "localhost:6379" # Password: "" # DB: 0 + # BoltDBOptions: + # FilePath: "./chains/privnet.bolt" RPCPort: 20335 NodePort: 20336 Relay: true diff --git a/config/protocol.privnet.docker.two.yml b/config/protocol.privnet.docker.two.yml index 449e64df4..422e452ed 100644 --- a/config/protocol.privnet.docker.two.yml +++ b/config/protocol.privnet.docker.two.yml @@ -15,7 +15,7 @@ ProtocolConfiguration: ApplicationConfiguration: DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/privnet" @@ -23,6 +23,8 @@ ApplicationConfiguration: # Addr: "localhost:6379" # Password: "" # DB: 0 + # BoltDBOptions: + # FilePath: "./chains/privnet.bolt" RPCPort: 20334 NodePort: 20335 Relay: true diff --git a/config/protocol.privnet.yml b/config/protocol.privnet.yml index 9e4242eab..9e28b12d6 100644 --- a/config/protocol.privnet.yml +++ b/config/protocol.privnet.yml @@ -21,7 +21,7 @@ ProtocolConfiguration: ApplicationConfiguration: DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/privnet" @@ -29,6 +29,8 @@ ApplicationConfiguration: # Addr: "localhost:6379" # Password: "" # DB: 0 + # BoltDBOptions: + # FilePath: "./chains/privnet.bolt" RPCPort: 20331 NodePort: 20332 Relay: true diff --git a/config/protocol.testnet.yml b/config/protocol.testnet.yml index 06dee5a8a..13b33a90d 100644 --- a/config/protocol.testnet.yml +++ b/config/protocol.testnet.yml @@ -30,7 +30,7 @@ ProtocolConfiguration: ApplicationConfiguration: DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/testnet" @@ -38,6 +38,8 @@ ApplicationConfiguration: # Addr: "localhost:6379" # Password: "" # DB: 0 + # BoltDBOptions: + # FilePath: "./chains/testnet.bolt" RPCPort: 20332 NodePort: 20333 Relay: true diff --git a/config/protocol.unit_testnet.yml b/config/protocol.unit_testnet.yml index 299440f03..eb3490e85 100644 --- a/config/protocol.unit_testnet.yml +++ b/config/protocol.unit_testnet.yml @@ -20,7 +20,7 @@ ProtocolConfiguration: ApplicationConfiguration: DBConfiguration: - Type: "inmemory" #other options: 'inmemory','redis'. + Type: "inmemory" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. # LevelDBOptions: # DataDirectoryPath: "./chains/unit_testnet" @@ -28,6 +28,8 @@ ApplicationConfiguration: # Addr: "localhost:6379" # Password: "" # DB: 0 + # BoltDBOptions: + # FilePath: "./chains/unit_testnet.bolt" RPCPort: 20332 NodePort: 20333 Relay: true diff --git a/go.mod b/go.mod index b59552d03..80515b6c0 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ require ( github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db // indirect github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect github.com/alicebob/miniredis v2.5.0+incompatible - github.com/davecgh/go-spew v1.1.0 // indirect + github.com/etcd-io/bbolt v1.3.3 github.com/fatih/color v1.7.0 // indirect github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect github.com/go-redis/redis v6.10.2+incompatible @@ -18,12 +18,13 @@ require ( github.com/nspcc-dev/rfc6979 v0.1.0 github.com/onsi/gomega v1.4.2 // indirect github.com/pkg/errors v0.8.0 - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sirupsen/logrus v1.0.5 - github.com/stretchr/testify v1.2.1 + github.com/stretchr/objx v0.2.0 // indirect + github.com/stretchr/testify v1.3.0 github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73 github.com/urfave/cli v1.20.0 github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 // indirect + go.etcd.io/bbolt v1.3.3 // indirect golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb golang.org/x/text v0.3.0 golang.org/x/tools v0.0.0-20180318012157-96caea41033d diff --git a/go.sum b/go.sum index 9e5af119e..d6ecfc850 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,10 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-redis/redis v6.10.2+incompatible h1:SLbqrO/Ik1nhXA5/cbEs1P5MUBo1Qq4ihlNfGnnipPw= @@ -50,14 +54,21 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73 h1:I2drr5K0tykBofr74ZEGliE/Hf6fNkEbcPyFvsy7wZk= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 h1:1b6PAtenNyhsmo/NKXVe34h7JEZKva1YB/ne7K7mqKM= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= +go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb h1:O6ztCaemiMr99EgJdgXrr0J7N0EQN1oky/0GxML9Avk= golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= diff --git a/pkg/core/storage/boltdb_store.go b/pkg/core/storage/boltdb_store.go new file mode 100644 index 000000000..b0c18c7c7 --- /dev/null +++ b/pkg/core/storage/boltdb_store.go @@ -0,0 +1,136 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "os" + "path" + + "github.com/etcd-io/bbolt" + "github.com/syndtr/goleveldb/leveldb/util" +) + +// BoltDBOptions configuration for boltdb. +type BoltDBOptions struct { + FilePath string `yaml:"FilePath"` +} + +// Bucket represents bucket used in boltdb to store all the data. +var Bucket = []byte("DB") + +// BoltDBStore it is the storage implementation for storing and retrieving +// blockchain data. +type BoltDBStore struct { + db *bbolt.DB +} + +// BoltDBBatch simple batch implementation to satisfy the Store interface. +type BoltDBBatch struct { + mem map[*[]byte][]byte +} + +// Len implements the Batch interface. +func (b *BoltDBBatch) Len() int { + return len(b.mem) +} + +// Put implements the Batch interface. +func (b *BoltDBBatch) Put(k, v []byte) { + b.mem[&k] = v +} + +// NewBoltDBStore returns a new ready to use BoltDB storage with created bucket. +func NewBoltDBStore(ctx context.Context, cfg BoltDBOptions) (*BoltDBStore, error) { + var opts *bbolt.Options // should be exposed via BoltDBOptions if anything needed + fileMode := os.FileMode(0600) // should be exposed via BoltDBOptions if anything needed + fileName := cfg.FilePath + dir := path.Dir(fileName) + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("could not create dir for BoltDB: %v", err) + } + db, err := bbolt.Open(fileName, fileMode, opts) + if err != nil { + return nil, err + } + err = db.Update(func(tx *bbolt.Tx) error { + _, err = tx.CreateBucketIfNotExists(Bucket) + if err != nil { + return fmt.Errorf("could not create root bucket: %v", err) + } + return nil + }) + + // graceful shutdown + go func() { + <-ctx.Done() + db.Close() + }() + + return &BoltDBStore{db: db}, nil +} + +// Put implements the Store interface. +func (s *BoltDBStore) Put(key, value []byte) error { + return s.db.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket(Bucket) + err := b.Put(key, value) + return err + }) +} + +// Get implements the Store interface. +func (s *BoltDBStore) Get(key []byte) (val []byte, err error) { + err = s.db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(Bucket) + val = b.Get(key) + return nil + }) + if val == nil { + err = ErrKeyNotFound + } + return +} + +// PutBatch implements the Store interface. +func (s *BoltDBStore) PutBatch(batch Batch) error { + return s.db.Batch(func(tx *bbolt.Tx) error { + b := tx.Bucket(Bucket) + for k, v := range batch.(*BoltDBBatch).mem { + err := b.Put(*k, v) + if err != nil { + return err + } + } + return nil + }) +} + +// Seek implements the Store interface. +func (s *BoltDBStore) Seek(key []byte, f func(k, v []byte)) { + err := s.db.View(func(tx *bbolt.Tx) error { + c := tx.Bucket(Bucket).Cursor() + prefix := util.BytesPrefix(key) + for k, v := c.Seek(prefix.Start); k != nil && bytes.Compare(k, prefix.Limit) <= 0; k, v = c.Next() { + f(k, v) + } + return nil + }) + if err != nil { + fmt.Println("error while executing seek in boltDB") + } +} + +// Batch implements the Batch interface and returns a boltdb +// compatible Batch. +func (s *BoltDBStore) Batch() Batch { + return &BoltDBBatch{ + mem: make(map[*[]byte][]byte), + } +} + +// Close releases all db resources. +func (s *BoltDBStore) Close() error { + return s.db.Close() +} diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index f5dfb3882..1873339b9 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -85,6 +85,8 @@ func NewStore(context context.Context, cfg DBConfiguration) (Store, error) { store = NewMemoryStore() case "redis": store, err = NewRedisStore(cfg.RedisDBOptions) + case "boltdb": + store, err = NewBoltDBStore(context, cfg.BoltDBOptions) } return store, err } diff --git a/pkg/core/storage/store_config.go b/pkg/core/storage/store_config.go index ebf1f2f92..272008be4 100644 --- a/pkg/core/storage/store_config.go +++ b/pkg/core/storage/store_config.go @@ -6,5 +6,6 @@ type ( Type string `yaml:"Type"` LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"` RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"` + BoltDBOptions BoltDBOptions `yaml:"BoltDBOptions"` } ) From 2cabd5b274e1f775100038c4c6140fd269042b36 Mon Sep 17 00:00:00 2001 From: Vsevolod Brekelov Date: Sat, 14 Sep 2019 10:29:08 +0300 Subject: [PATCH 2/2] storage: add boltDB unit tests --- pkg/core/storage/boltdb_store_test.go | 89 +++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 pkg/core/storage/boltdb_store_test.go diff --git a/pkg/core/storage/boltdb_store_test.go b/pkg/core/storage/boltdb_store_test.go new file mode 100644 index 000000000..73dc4097d --- /dev/null +++ b/pkg/core/storage/boltdb_store_test.go @@ -0,0 +1,89 @@ +package storage + +import ( + "context" + "io/ioutil" + "os" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBoltDBBatch(t *testing.T) { + boltDB := BoltDBStore{} + want := &BoltDBBatch{mem: map[*[]byte][]byte{}} + if got := boltDB.Batch(); !reflect.DeepEqual(got, want) { + t.Errorf("BoltDB Batch() = %v, want %v", got, want) + } +} + +func TestBoltDBBatch_Len(t *testing.T) { + batch := &BoltDBBatch{mem: map[*[]byte][]byte{}} + want := len(map[*[]byte][]byte{}) + assert.Equal(t, want, batch.Len()) +} + +func TestBoltDBBatch_PutBatchAndGet(t *testing.T) { + key := []byte("foo") + value := []byte("bar") + batch := &BoltDBBatch{mem: map[*[]byte][]byte{&key: value}} + + boltDBStore := openStore(t) + + errPut := boltDBStore.PutBatch(batch) + assert.Nil(t, errPut, "Error while PutBatch") + + result, err := boltDBStore.Get(key) + assert.Nil(t, err) + assert.Equal(t, value, result) + + require.NoError(t, boltDBStore.Close()) +} + +func TestBoltDBBatch_PutAndGet(t *testing.T) { + key := []byte("foo") + value := []byte("bar") + + boltDBStore := openStore(t) + + errPut := boltDBStore.Put(key, value) + assert.Nil(t, errPut, "Error while Put") + + result, err := boltDBStore.Get(key) + assert.Nil(t, err) + assert.Equal(t, value, result) + + require.NoError(t, boltDBStore.Close()) +} + +func TestBoltDBStore_Seek(t *testing.T) { + key := []byte("foo") + value := []byte("bar") + + boltDBStore := openStore(t) + + errPut := boltDBStore.Put(key, value) + assert.Nil(t, errPut, "Error while Put") + + boltDBStore.Seek(key, func(k, v []byte) { + assert.Equal(t, value, v) + }) + + require.NoError(t, boltDBStore.Close()) +} + +func openStore(t *testing.T) *BoltDBStore { + testFileName := "test_bolt_db" + file, err := ioutil.TempFile("", testFileName) + defer func() { + err := os.RemoveAll(testFileName) + require.NoError(t, err) + }() + require.NoError(t, err) + require.NoError(t, file.Close()) + boltDBStore, err := NewBoltDBStore(context.Background(), BoltDBOptions{FilePath: testFileName}) + require.NoError(t, err) + return boltDBStore +}