From fb4b87bb96a9de64fffea64e86cb44712762953d Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 18 Aug 2021 15:36:09 +0300 Subject: [PATCH 1/2] storage: drop BadgerDB support, close #2130 --- config/protocol.mainnet.yml | 4 +- config/protocol.privnet.docker.four.yml | 4 +- config/protocol.privnet.docker.one.yml | 4 +- config/protocol.privnet.docker.single.yml | 4 +- config/protocol.privnet.docker.three.yml | 4 +- config/protocol.privnet.docker.two.yml | 4 +- config/protocol.privnet.yml | 4 +- config/protocol.testnet.yml | 4 +- config/protocol.unit_testnet.single.yml | 4 +- config/protocol.unit_testnet.yml | 4 +- docs/node-configuration.md | 5 +- go.mod | 1 - go.sum | 6 - pkg/core/storage/badgerdb_store.go | 155 ---------------------- pkg/core/storage/badgerdb_store_test.go | 20 --- pkg/core/storage/store.go | 2 - pkg/core/storage/store_config.go | 11 +- pkg/core/storage/storeandbatch_test.go | 1 - 18 files changed, 16 insertions(+), 225 deletions(-) delete mode 100644 pkg/core/storage/badgerdb_store.go delete mode 100644 pkg/core/storage/badgerdb_store_test.go diff --git a/config/protocol.mainnet.yml b/config/protocol.mainnet.yml index 19d29973c..4c037ba81 100644 --- a/config/protocol.mainnet.yml +++ b/config/protocol.mainnet.yml @@ -51,7 +51,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/mainnet" @@ -61,8 +61,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/mainnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/mainnet.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 10333 diff --git a/config/protocol.privnet.docker.four.yml b/config/protocol.privnet.docker.four.yml index 9abac56cc..40bb35720 100644 --- a/config/protocol.privnet.docker.four.yml +++ b/config/protocol.privnet.docker.four.yml @@ -32,7 +32,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/four" @@ -42,8 +42,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/four.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20336 diff --git a/config/protocol.privnet.docker.one.yml b/config/protocol.privnet.docker.one.yml index 558069494..ced25975e 100644 --- a/config/protocol.privnet.docker.one.yml +++ b/config/protocol.privnet.docker.one.yml @@ -32,7 +32,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/one" @@ -42,8 +42,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/one.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20333 diff --git a/config/protocol.privnet.docker.single.yml b/config/protocol.privnet.docker.single.yml index 8b1de6ef4..39a0efa91 100644 --- a/config/protocol.privnet.docker.single.yml +++ b/config/protocol.privnet.docker.single.yml @@ -26,7 +26,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/single" @@ -36,8 +36,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/single.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20333 diff --git a/config/protocol.privnet.docker.three.yml b/config/protocol.privnet.docker.three.yml index 2cdd77b2c..5ecc545ff 100644 --- a/config/protocol.privnet.docker.three.yml +++ b/config/protocol.privnet.docker.three.yml @@ -32,7 +32,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/three" @@ -42,8 +42,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/three.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20335 diff --git a/config/protocol.privnet.docker.two.yml b/config/protocol.privnet.docker.two.yml index 92b0f868a..f5aaab4ff 100644 --- a/config/protocol.privnet.docker.two.yml +++ b/config/protocol.privnet.docker.two.yml @@ -32,7 +32,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/two" @@ -42,8 +42,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/two.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20334 diff --git a/config/protocol.privnet.yml b/config/protocol.privnet.yml index 6f54b307c..2e9c5865a 100644 --- a/config/protocol.privnet.yml +++ b/config/protocol.privnet.yml @@ -32,7 +32,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/privnet" @@ -42,8 +42,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/privnet.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20332 diff --git a/config/protocol.testnet.yml b/config/protocol.testnet.yml index 51d86d94c..a606aac76 100644 --- a/config/protocol.testnet.yml +++ b/config/protocol.testnet.yml @@ -51,7 +51,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "leveldb" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/testnet" @@ -61,8 +61,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/testnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/testnet.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20333 diff --git a/config/protocol.unit_testnet.single.yml b/config/protocol.unit_testnet.single.yml index 1e565796a..d73a99208 100644 --- a/config/protocol.unit_testnet.single.yml +++ b/config/protocol.unit_testnet.single.yml @@ -25,7 +25,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "inmemory" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "inmemory" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. # LevelDBOptions: # DataDirectoryPath: "./chains/unit_testnet" @@ -35,8 +35,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/unit_testnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/unit_testnet.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 0 diff --git a/config/protocol.unit_testnet.yml b/config/protocol.unit_testnet.yml index 2f4b1e1d0..0db5d20de 100644 --- a/config/protocol.unit_testnet.yml +++ b/config/protocol.unit_testnet.yml @@ -34,7 +34,7 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "inmemory" #other options: 'inmemory','redis','boltdb', 'badgerdb'. + Type: "inmemory" #other options: 'inmemory','redis','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. # LevelDBOptions: # DataDirectoryPath: "./chains/unit_testnet" @@ -44,8 +44,6 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/unit_testnet.bolt" - # BadgerDBOptions: - # BadgerDir: "./chains/unit_testnet.badger" # Uncomment in order to set up custom address for node. # Address: 127.0.0.1 NodePort: 20333 diff --git a/docs/node-configuration.md b/docs/node-configuration.md index 1fb8cfdef..53ab4fceb 100644 --- a/docs/node-configuration.md +++ b/docs/node-configuration.md @@ -53,16 +53,13 @@ DBConfiguration: DB: 0 BoltDBOptions: FilePath: ./chains/privnet.bolt - BadgerDBOptions: - BadgerDir: ./chains/privnet.badger ``` where: - `Type` is the database type (string value). Supported types: `levelDB`, - `redisDB`, `boltDB`, `badgerDB`. + `redisDB`, `boltDB` - `LevelDBOptions` are settings for LevelDB. - `RedisDBOptions` are options for RedisDB. - `BoltDBOptions` configures BoltDB. -- `BadgerDBOptions` are options for BadgerDB. Only options for the specified database type will be used. diff --git a/go.mod b/go.mod index 26b6d1790..37ac942d8 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db github.com/alicebob/miniredis/v2 v2.15.1 github.com/btcsuite/btcd v0.22.0-beta - github.com/dgraph-io/badger/v2 v2.0.3 github.com/go-redis/redis v6.15.9+incompatible github.com/gorilla/websocket v1.4.2 github.com/hashicorp/golang-lru v0.5.4 diff --git a/go.sum b/go.sum index 15deb0983..e75be0cf0 100644 --- a/go.sum +++ b/go.sum @@ -5,9 +5,7 @@ github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1 github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= -github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/Workiva/go-datastructures v1.0.53 h1:J6Y/52yX10Xc5JjXmGtWoSSxs3mZnGSaq37xZZh7Yig= @@ -76,13 +74,9 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI= github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= -github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= diff --git a/pkg/core/storage/badgerdb_store.go b/pkg/core/storage/badgerdb_store.go deleted file mode 100644 index 6cae62849..000000000 --- a/pkg/core/storage/badgerdb_store.go +++ /dev/null @@ -1,155 +0,0 @@ -package storage - -import ( - "os" - - "github.com/dgraph-io/badger/v2" - "github.com/nspcc-dev/neo-go/pkg/util/slice" -) - -// BadgerDBOptions configuration for BadgerDB. -type BadgerDBOptions struct { - Dir string `yaml:"BadgerDir"` -} - -// BadgerDBStore is the official storage implementation for storing and retrieving -// blockchain data. -type BadgerDBStore struct { - db *badger.DB -} - -// BadgerDBBatch is a wrapper around badger.WriteBatch, compatible with Batch interface. -type BadgerDBBatch struct { - batch *badger.WriteBatch -} - -// Delete implements the Batch interface. -func (b *BadgerDBBatch) Delete(key []byte) { - err := b.batch.Delete(key) - if err != nil { - panic(err) - } -} - -// Put implements the Batch interface. -func (b *BadgerDBBatch) Put(key, value []byte) { - keycopy := slice.Copy(key) - valuecopy := slice.Copy(value) - err := b.batch.Set(keycopy, valuecopy) - if err != nil { - panic(err) - } -} - -// NewBadgerDBStore returns a new BadgerDBStore object that will -// initialize the database found at the given path. -func NewBadgerDBStore(cfg BadgerDBOptions) (*BadgerDBStore, error) { - // BadgerDB isn't able to make nested directories - err := os.MkdirAll(cfg.Dir, os.ModePerm) - if err != nil { - panic(err) - } - opts := badger.DefaultOptions(cfg.Dir) // should be exposed via BadgerDBOptions if anything needed - - db, err := badger.Open(opts) - if err != nil { - return nil, err - } - - return &BadgerDBStore{ - db: db, - }, nil -} - -// Batch implements the Batch interface and returns a badgerdb -// compatible Batch. -func (b *BadgerDBStore) Batch() Batch { - return &BadgerDBBatch{b.db.NewWriteBatch()} -} - -// Delete implements the Store interface. -func (b *BadgerDBStore) Delete(key []byte) error { - return b.db.Update(func(txn *badger.Txn) error { - return txn.Delete(key) - }) -} - -// Get implements the Store interface. -func (b *BadgerDBStore) Get(key []byte) ([]byte, error) { - var val []byte - err := b.db.View(func(txn *badger.Txn) error { - item, err := txn.Get(key) - if err == badger.ErrKeyNotFound { - return ErrKeyNotFound - } - val, err = item.ValueCopy(nil) - return err - }) - return val, err -} - -// Put implements the Store interface. -func (b *BadgerDBStore) Put(key, value []byte) error { - return b.db.Update(func(txn *badger.Txn) error { - err := txn.Set(key, value) - return err - }) -} - -// PutBatch implements the Store interface. -func (b *BadgerDBStore) PutBatch(batch Batch) error { - defer batch.(*BadgerDBBatch).batch.Cancel() - return batch.(*BadgerDBBatch).batch.Flush() -} - -// PutChangeSet implements the Store interface. -func (b *BadgerDBStore) PutChangeSet(puts map[string][]byte, dels map[string]bool) error { - return b.db.Update(func(txn *badger.Txn) error { - for k, v := range puts { - err := txn.Set([]byte(k), v) - if err != nil { - return err - } - } - for k := range dels { - err := txn.Delete([]byte(k)) - if err != nil { - return err - } - } - return nil - }) -} - -// Seek implements the Store interface. -func (b *BadgerDBStore) Seek(key []byte, f func(k, v []byte)) { - err := b.db.View(func(txn *badger.Txn) error { - it := txn.NewIterator(badger.IteratorOptions{ - PrefetchValues: true, - PrefetchSize: 100, - Reverse: false, - AllVersions: false, - Prefix: key, - InternalAccess: false, - }) - defer it.Close() - for it.Seek(key); it.ValidForPrefix(key); it.Next() { - item := it.Item() - k := item.Key() - v, err := item.ValueCopy(nil) - if err != nil { - return err - } - f(k, v) - } - return nil - }) - if err != nil { - panic(err) - } -} - -// Close releases all db resources. -func (b *BadgerDBStore) Close() error { - return b.db.Close() -} diff --git a/pkg/core/storage/badgerdb_store_test.go b/pkg/core/storage/badgerdb_store_test.go deleted file mode 100644 index 78a8fb957..000000000 --- a/pkg/core/storage/badgerdb_store_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package storage - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func newBadgerDBForTesting(t testing.TB) Store { - bdbDir := t.TempDir() - dbConfig := DBConfiguration{ - Type: "badgerdb", - BadgerDBOptions: BadgerDBOptions{ - Dir: bdbDir, - }, - } - newBadgerStore, err := NewBadgerDBStore(dbConfig.BadgerDBOptions) - require.Nil(t, err, "NewBadgerDBStore error") - return newBadgerStore -} diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index f87f94e87..cad4613b1 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -110,8 +110,6 @@ func NewStore(cfg DBConfiguration) (Store, error) { store, err = NewRedisStore(cfg.RedisDBOptions) case "boltdb": store, err = NewBoltDBStore(cfg.BoltDBOptions) - case "badgerdb": - store, err = NewBadgerDBStore(cfg.BadgerDBOptions) default: return nil, fmt.Errorf("unknown storage: %s", cfg.Type) } diff --git a/pkg/core/storage/store_config.go b/pkg/core/storage/store_config.go index 34a65fc99..ac3d228cd 100644 --- a/pkg/core/storage/store_config.go +++ b/pkg/core/storage/store_config.go @@ -1,12 +1,11 @@ package storage type ( - // DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB', 'boltDB', 'badgerDB'. + // DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB', 'boltDB'. DBConfiguration struct { - Type string `yaml:"Type"` - LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"` - RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"` - BoltDBOptions BoltDBOptions `yaml:"BoltDBOptions"` - BadgerDBOptions BadgerDBOptions `yaml:"BadgerDBOptions"` + Type string `yaml:"Type"` + LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"` + RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"` + BoltDBOptions BoltDBOptions `yaml:"BoltDBOptions"` } ) diff --git a/pkg/core/storage/storeandbatch_test.go b/pkg/core/storage/storeandbatch_test.go index 70a3787ea..9d01e0c29 100644 --- a/pkg/core/storage/storeandbatch_test.go +++ b/pkg/core/storage/storeandbatch_test.go @@ -223,7 +223,6 @@ func TestAllDBs(t *testing.T) { {"MemCached", newMemCachedStoreForTesting}, {"Memory", newMemoryStoreForTesting}, {"RedisDB", newRedisStoreForTesting}, - {"BadgerDB", newBadgerDBForTesting}, } var tests = []dbTestFunction{testStoreClose, testStorePutAndGet, testStoreGetNonExistent, testStorePutBatch, testStoreSeek, From 1144a03486b3cf0d2032309061295a947c1f60f4 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 18 Aug 2021 15:39:12 +0300 Subject: [PATCH 2/2] storage: drop RedisDB, close #2130 --- config/protocol.mainnet.yml | 6 +- config/protocol.privnet.docker.four.yml | 6 +- config/protocol.privnet.docker.one.yml | 6 +- config/protocol.privnet.docker.single.yml | 6 +- config/protocol.privnet.docker.three.yml | 6 +- config/protocol.privnet.docker.two.yml | 6 +- config/protocol.privnet.yml | 6 +- config/protocol.testnet.yml | 6 +- config/protocol.unit_testnet.single.yml | 6 +- config/protocol.unit_testnet.yml | 6 +- docs/node-configuration.md | 9 +-- go.mod | 2 - go.sum | 9 --- pkg/core/storage/redis_store.go | 95 ----------------------- pkg/core/storage/redis_store_test.go | 44 ----------- pkg/core/storage/store.go | 2 - pkg/core/storage/store_config.go | 3 +- pkg/core/storage/storeandbatch_test.go | 1 - 18 files changed, 13 insertions(+), 212 deletions(-) delete mode 100644 pkg/core/storage/redis_store.go delete mode 100644 pkg/core/storage/redis_store_test.go diff --git a/config/protocol.mainnet.yml b/config/protocol.mainnet.yml index 4c037ba81..19823ac16 100644 --- a/config/protocol.mainnet.yml +++ b/config/protocol.mainnet.yml @@ -51,14 +51,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb' + Type: "leveldb" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/mainnet" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/mainnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/config/protocol.privnet.docker.four.yml b/config/protocol.privnet.docker.four.yml index 40bb35720..365b90c92 100644 --- a/config/protocol.privnet.docker.four.yml +++ b/config/protocol.privnet.docker.four.yml @@ -32,14 +32,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb' + Type: "leveldb" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/four" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/config/protocol.privnet.docker.one.yml b/config/protocol.privnet.docker.one.yml index ced25975e..736cf0ad9 100644 --- a/config/protocol.privnet.docker.one.yml +++ b/config/protocol.privnet.docker.one.yml @@ -32,14 +32,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb' + Type: "leveldb" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/one" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/config/protocol.privnet.docker.single.yml b/config/protocol.privnet.docker.single.yml index 39a0efa91..10a6602c6 100644 --- a/config/protocol.privnet.docker.single.yml +++ b/config/protocol.privnet.docker.single.yml @@ -26,14 +26,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb' + Type: "leveldb" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/single" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/config/protocol.privnet.docker.three.yml b/config/protocol.privnet.docker.three.yml index 5ecc545ff..dccec4aea 100644 --- a/config/protocol.privnet.docker.three.yml +++ b/config/protocol.privnet.docker.three.yml @@ -32,14 +32,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb' + Type: "leveldb" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/three" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/config/protocol.privnet.docker.two.yml b/config/protocol.privnet.docker.two.yml index f5aaab4ff..7f6fca93a 100644 --- a/config/protocol.privnet.docker.two.yml +++ b/config/protocol.privnet.docker.two.yml @@ -32,14 +32,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb' + Type: "leveldb" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "/chains/two" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/config/protocol.privnet.yml b/config/protocol.privnet.yml index 2e9c5865a..5756eee92 100644 --- a/config/protocol.privnet.yml +++ b/config/protocol.privnet.yml @@ -32,14 +32,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb' + Type: "leveldb" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/privnet" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/config/protocol.testnet.yml b/config/protocol.testnet.yml index a606aac76..a57cda008 100644 --- a/config/protocol.testnet.yml +++ b/config/protocol.testnet.yml @@ -51,14 +51,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','redis','boltdb' + Type: "leveldb" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/testnet" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/testnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/config/protocol.unit_testnet.single.yml b/config/protocol.unit_testnet.single.yml index d73a99208..ef6f97bfb 100644 --- a/config/protocol.unit_testnet.single.yml +++ b/config/protocol.unit_testnet.single.yml @@ -25,14 +25,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "inmemory" #other options: 'inmemory','redis','boltdb' + Type: "inmemory" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. # LevelDBOptions: # DataDirectoryPath: "./chains/unit_testnet" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/unit_testnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/config/protocol.unit_testnet.yml b/config/protocol.unit_testnet.yml index 0db5d20de..bd65fe04e 100644 --- a/config/protocol.unit_testnet.yml +++ b/config/protocol.unit_testnet.yml @@ -34,14 +34,10 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "inmemory" #other options: 'inmemory','redis','boltdb' + Type: "inmemory" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. # LevelDBOptions: # DataDirectoryPath: "./chains/unit_testnet" - # RedisDBOptions: - # Addr: "localhost:6379" - # Password: "" - # DB: 0 # BoltDBOptions: # FilePath: "./chains/unit_testnet.bolt" # Uncomment in order to set up custom address for node. diff --git a/docs/node-configuration.md b/docs/node-configuration.md index 53ab4fceb..dbbd5dcb3 100644 --- a/docs/node-configuration.md +++ b/docs/node-configuration.md @@ -47,18 +47,13 @@ DBConfiguration: Type: leveldb LevelDBOptions: DataDirectoryPath: /chains/privnet - RedisDBOptions: - Addr: localhost:6379 - Password: "" - DB: 0 BoltDBOptions: FilePath: ./chains/privnet.bolt ``` where: -- `Type` is the database type (string value). Supported types: `levelDB`, - `redisDB`, `boltDB` +- `Type` is the database type (string value). Supported types: `levelDB` and + `boltDB`. - `LevelDBOptions` are settings for LevelDB. -- `RedisDBOptions` are options for RedisDB. - `BoltDBOptions` configures BoltDB. Only options for the specified database type will be used. diff --git a/go.mod b/go.mod index 37ac942d8..2f5d4537b 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,7 @@ require ( github.com/Workiva/go-datastructures v1.0.53 github.com/abiosoft/ishell/v2 v2.0.2 github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db - github.com/alicebob/miniredis/v2 v2.15.1 github.com/btcsuite/btcd v0.22.0-beta - github.com/go-redis/redis v6.15.9+incompatible github.com/gorilla/websocket v1.4.2 github.com/hashicorp/golang-lru v0.5.4 github.com/mr-tron/base58 v1.2.0 diff --git a/go.sum b/go.sum index e75be0cf0..5ca0ba5c2 100644 --- a/go.sum +++ b/go.sum @@ -23,12 +23,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= -github.com/alicebob/miniredis/v2 v2.15.1 h1:Fw+ixAJPmKhCLBqDwHlTDqxUxp0xjEwXczEpt1B6r7k= -github.com/alicebob/miniredis/v2 v2.15.1/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODVXVpYDRaGhWaL6I= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -99,8 +94,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= -github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -297,8 +290,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= -github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg= -github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= diff --git a/pkg/core/storage/redis_store.go b/pkg/core/storage/redis_store.go deleted file mode 100644 index b1631068b..000000000 --- a/pkg/core/storage/redis_store.go +++ /dev/null @@ -1,95 +0,0 @@ -package storage - -import ( - "fmt" - - "github.com/go-redis/redis" -) - -// RedisDBOptions configuration for RedisDB. -type RedisDBOptions struct { - Addr string `yaml:"Addr"` - Password string `yaml:"Password"` - DB int `yaml:"DB"` -} - -// RedisStore holds the client and maybe later some more metadata. -type RedisStore struct { - client *redis.Client -} - -// NewRedisStore returns an new initialized - ready to use RedisStore object. -func NewRedisStore(cfg RedisDBOptions) (*RedisStore, error) { - c := redis.NewClient(&redis.Options{ - Addr: cfg.Addr, - Password: cfg.Password, - DB: cfg.DB, - }) - if _, err := c.Ping().Result(); err != nil { - return nil, err - } - return &RedisStore{client: c}, nil -} - -// Batch implements the Store interface. -func (s *RedisStore) Batch() Batch { - return newMemoryBatch() -} - -// Get implements the Store interface. -func (s *RedisStore) Get(k []byte) ([]byte, error) { - val, err := s.client.Get(string(k)).Result() - if err != nil { - if err == redis.Nil { - err = ErrKeyNotFound - } - return nil, err - } - return []byte(val), nil -} - -// Delete implements the Store interface. -func (s *RedisStore) Delete(k []byte) error { - s.client.Del(string(k)) - return nil -} - -// Put implements the Store interface. -func (s *RedisStore) Put(k, v []byte) error { - s.client.Set(string(k), string(v), 0) - return nil -} - -// PutBatch implements the Store interface. -func (s *RedisStore) PutBatch(b Batch) error { - memBatch := b.(*MemoryBatch) - return s.PutChangeSet(memBatch.mem, memBatch.del) -} - -// PutChangeSet implements the Store interface. -func (s *RedisStore) PutChangeSet(puts map[string][]byte, dels map[string]bool) error { - pipe := s.client.Pipeline() - for k, v := range puts { - pipe.Set(k, v, 0) - } - for k := range dels { - pipe.Del(k) - } - _, err := pipe.Exec() - return err -} - -// Seek implements the Store interface. -func (s *RedisStore) Seek(k []byte, f func(k, v []byte)) { - iter := s.client.Scan(0, fmt.Sprintf("%s*", k), 0).Iterator() - for iter.Next() { - key := iter.Val() - val, _ := s.client.Get(key).Result() - f([]byte(key), []byte(val)) - } -} - -// Close implements the Store interface. -func (s *RedisStore) Close() error { - return s.client.Close() -} diff --git a/pkg/core/storage/redis_store_test.go b/pkg/core/storage/redis_store_test.go deleted file mode 100644 index f18aca684..000000000 --- a/pkg/core/storage/redis_store_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package storage - -import ( - "testing" - - "github.com/alicebob/miniredis/v2" - "github.com/stretchr/testify/require" -) - -type mockedRedisStore struct { - RedisStore - mini *miniredis.Miniredis -} - -func prepareRedisMock(t testing.TB) (*miniredis.Miniredis, *RedisStore) { - miniRedis, err := miniredis.Run() - require.Nil(t, err, "MiniRedis mock creation error") - - _ = miniRedis.Set("foo", "bar") - - dbConfig := DBConfiguration{ - Type: "redisDB", - RedisDBOptions: RedisDBOptions{ - Addr: miniRedis.Addr(), - Password: "", - DB: 0, - }, - } - newRedisStore, err := NewRedisStore(dbConfig.RedisDBOptions) - require.Nil(t, err, "NewRedisStore() error") - return miniRedis, newRedisStore -} - -func (mrs *mockedRedisStore) Close() error { - err := mrs.RedisStore.Close() - mrs.mini.Close() - return err -} - -func newRedisStoreForTesting(t testing.TB) Store { - mock, rs := prepareRedisMock(t) - mrs := &mockedRedisStore{RedisStore: *rs, mini: mock} - return mrs -} diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index cad4613b1..0888b9d38 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -106,8 +106,6 @@ func NewStore(cfg DBConfiguration) (Store, error) { store, err = NewLevelDBStore(cfg.LevelDBOptions) case "inmemory": store = NewMemoryStore() - case "redis": - store, err = NewRedisStore(cfg.RedisDBOptions) case "boltdb": store, err = NewBoltDBStore(cfg.BoltDBOptions) default: diff --git a/pkg/core/storage/store_config.go b/pkg/core/storage/store_config.go index ac3d228cd..3521e32a1 100644 --- a/pkg/core/storage/store_config.go +++ b/pkg/core/storage/store_config.go @@ -1,11 +1,10 @@ package storage type ( - // DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB', 'boltDB'. + // DBConfiguration describes configuration for DB. Supported: 'levelDB', 'boltDB'. DBConfiguration struct { Type string `yaml:"Type"` LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"` - RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"` BoltDBOptions BoltDBOptions `yaml:"BoltDBOptions"` } ) diff --git a/pkg/core/storage/storeandbatch_test.go b/pkg/core/storage/storeandbatch_test.go index 9d01e0c29..a8591ddfe 100644 --- a/pkg/core/storage/storeandbatch_test.go +++ b/pkg/core/storage/storeandbatch_test.go @@ -222,7 +222,6 @@ func TestAllDBs(t *testing.T) { {"LevelDB", newLevelDBForTesting}, {"MemCached", newMemCachedStoreForTesting}, {"Memory", newMemoryStoreForTesting}, - {"RedisDB", newRedisStoreForTesting}, } var tests = []dbTestFunction{testStoreClose, testStorePutAndGet, testStoreGetNonExistent, testStorePutBatch, testStoreSeek,