TrueCloudLab/neo-go
8
0
Fork 0
mirror of https://github.com/nspcc-dev/neo-go.git synced 2024-12-23 03:41:34 +00:00
Code Issues Projects Releases Packages Wiki Activity

storage: multiple DB support and Redis support

Browse source 
add config which closes #336
add redis db support
This commit is contained in:
Vsevolod Brekelov 2019-09-10 17:22:21 +03:00
parent 0055701ec0
commit 4f680703a4
19 changed files with 174 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
cli/server/server.go
View file

@ -8,6 +8,7 @@ import (
"github.com/CityOfZion/neo-go/config"
"github.com/CityOfZion/neo-go/pkg/core"
"github.com/CityOfZion/neo-go/pkg/core/storage"
"github.com/CityOfZion/neo-go/pkg/network"
"github.com/CityOfZion/neo-go/pkg/rpc"
"github.com/pkg/errors"
@ -64,10 +65,10 @@ func startServer(ctx *cli.Context) error {
}
serverConfig := network.NewServerConfig(cfg)
chain, err := core.NewBlockchainLevelDB(grace, cfg)
chain, err := initBlockChain(grace, cfg)
if err != nil {
err = fmt.Errorf("could not initialize blockchain: %s", err)
return cli.NewExitError(err, 1)
return err
}
if ctx.Bool("debug") {
@ -109,6 +110,20 @@ Main:
return nil
}
// initBlockChain initializes BlockChain with preselected DB.
func initBlockChain(context context.Context, cfg config.Config) (*core.Blockchain, error) {
store, err := storage.NewStore(context, cfg.ApplicationConfiguration.DBConfiguration)
if err != nil {
return nil, cli.NewExitError(fmt.Errorf("could not initialize storage: %s", err), 1)
}
chain, err := core.NewBlockchain(context, store, cfg.ProtocolConfiguration)
if err != nil {
return nil, cli.NewExitError(fmt.Errorf("could not initialize blockchain: %s", err), 1)
}
return chain, nil
}
func logo() string {
return `
_ ____________ __________

15
config/config.go
View file

@ -6,6 +6,7 @@ import (
"os"
"time"
"github.com/CityOfZion/neo-go/pkg/core/storage"
"github.com/CityOfZion/neo-go/pkg/core/transaction"
"github.com/CityOfZion/neo-go/pkg/util"
"github.com/go-yaml/yaml"
@ -60,13 +61,13 @@ type (
// ApplicationConfiguration config specific to the node.
ApplicationConfiguration struct {
DataDirectoryPath string `yaml:"DataDirectoryPath"`
RPCPort uint16 `yaml:"RPCPort"`
NodePort uint16 `yaml:"NodePort"`
Relay bool `yaml:"Relay"`
DialTimeout time.Duration `yaml:"DialTimeout"`
ProtoTickInterval time.Duration `yaml:"ProtoTickInterval"`
MaxPeers int `yaml:"MaxPeers"`
DBConfiguration storage.DBConfiguration `yaml:"DBConfiguration"`
RPCPort uint16 `yaml:"RPCPort"`
NodePort uint16 `yaml:"NodePort"`
Relay bool `yaml:"Relay"`
DialTimeout time.Duration `yaml:"DialTimeout"`
ProtoTickInterval time.Duration `yaml:"ProtoTickInterval"`
MaxPeers int `yaml:"MaxPeers"`
}
// NetMode describes the mode the blockchain will operate on.

10
config/protocol.mainnet.yml
View file

@ -24,7 +24,15 @@ ProtocolConfiguration:
RegisterTransaction: 10000
ApplicationConfiguration:
DataDirectoryPath: "./chains/mainnet"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/mainnet"
# RedisDBOptions:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
RPCPort: 20332
NodePort: 20333
Relay: true

10
config/protocol.privnet.docker.four.yml
View file

@ -17,7 +17,15 @@ ProtocolConfiguration:
RegisterTransaction: 10000
ApplicationConfiguration:
DataDirectoryPath: "/chains/privnet"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
# RedisDBOptions:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
RPCPort: 20336
NodePort: 20337
Relay: true

10
config/protocol.privnet.docker.one.yml
View file

@ -14,7 +14,15 @@ ProtocolConfiguration:
RegisterTransaction: 10000
ApplicationConfiguration:
DataDirectoryPath: "/chains/privnet"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
# RedisDBOptions:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
RPCPort: 20333
NodePort: 20334
Relay: true

10
config/protocol.privnet.docker.three.yml
View file

@ -14,7 +14,15 @@ ProtocolConfiguration:
RegisterTransaction: 10000
ApplicationConfiguration:
DataDirectoryPath: "/chains/privnet"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
# RedisDBOptions:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
RPCPort: 20335
NodePort: 20336
Relay: true

10
config/protocol.privnet.docker.two.yml
View file

@ -14,7 +14,15 @@ ProtocolConfiguration:
RegisterTransaction: 10000
ApplicationConfiguration:
DataDirectoryPath: "/chains/privnet"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
# RedisDBOptions:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
RPCPort: 20334
NodePort: 20335
Relay: true

10
config/protocol.privnet.yml
View file

@ -20,7 +20,15 @@ ProtocolConfiguration:
RegisterTransaction: 10000
ApplicationConfiguration:
DataDirectoryPath: "./chains/privnet"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/privnet"
# RedisDBOptions:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
RPCPort: 20331
NodePort: 20332
Relay: true

10
config/protocol.testnet.yml
View file

@ -24,7 +24,15 @@ ProtocolConfiguration:
RegisterTransaction: 100
ApplicationConfiguration:
DataDirectoryPath: "./chains/testnet"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/testnet"
# RedisDBOptions:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
RPCPort: 20332
NodePort: 20333
Relay: true

10
config/protocol.unit_testnet.yml
View file

@ -19,7 +19,15 @@ ProtocolConfiguration:
RegisterTransaction: 10000
ApplicationConfiguration:
DataDirectoryPath: "./chains/unit_testnet"
DBConfiguration:
Type: "leveldb" #other options: 'inmemory','redis'.
# DB type options. Uncomment those you need in case you want to switch DB type.
LevelDBOptions:
DataDirectoryPath: "./chains/unit_testnet"
# RedisDBOptions:
# Addr: "localhost:6379"
# Password: ""
# DB: 0
RPCPort: 20332
NodePort: 20333
Relay: true

4
go.mod
View file

@ -1,10 +1,13 @@
module github.com/CityOfZion/neo-go
require (
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect
github.com/alicebob/miniredis v2.5.0+incompatible
github.com/davecgh/go-spew v1.1.0 // indirect
github.com/go-redis/redis v6.10.2+incompatible
github.com/go-yaml/yaml v2.1.0+incompatible
github.com/golang/snappy v0.0.0-20170215233205-553a64147049 // indirect
github.com/gomodule/redigo v2.0.0+incompatible // indirect
github.com/mr-tron/base58 v1.1.2
github.com/nspcc-dev/rfc6979 v0.1.0
github.com/onsi/gomega v1.4.2 // indirect
@ -14,6 +17,7 @@ require (
github.com/stretchr/testify v1.2.1
github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73
github.com/urfave/cli v1.20.0
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 // indirect
golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb
golang.org/x/text v0.3.0
golang.org/x/tools v0.0.0-20180318012157-96caea41033d

13
go.sum
View file

@ -1,3 +1,10 @@
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U=
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI=
github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@ -10,6 +17,8 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20170215233205-553a64147049 h1:K9KHZbXKpGydfDN0aZrsoHpLJlZsBrGMFWbgLDGnPZk=
github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78=
@ -32,6 +41,8 @@ github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73 h1:I2drr5K0tykBof
github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 h1:1b6PAtenNyhsmo/NKXVe34h7JEZKva1YB/ne7K7mqKM=
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb h1:O6ztCaemiMr99EgJdgXrr0J7N0EQN1oky/0GxML9Avk=
golang.org/x/crypto v0.0.0-20180316180149-374053ea96cb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
@ -40,6 +51,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6Zh
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 h1:FDfvYgoVsA7TTZSbgiqjAbfPbK47CNHdWl3h/PJtii0=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20180318012157-96caea41033d h1:Xmo0nLTRYewf0eXDvo12nMSuOgNQ4283hdbOHIUf7h8=

14
pkg/core/blockchain.go
View file

@ -83,20 +83,6 @@ func NewBlockchain(ctx context.Context, s storage.Store, cfg config.ProtocolConf
return bc, nil
}
// NewBlockchainLevelDB initializes new blockchain DB store based on configuration
func NewBlockchainLevelDB(ctx context.Context, cfg config.Config) (*Blockchain, error) {
store, err := storage.NewLevelDBStore(
ctx,
cfg.ApplicationConfiguration.DataDirectoryPath,
nil,
)
if err != nil {
return nil, err
}
return NewBlockchain(ctx, store, cfg.ProtocolConfiguration)
}
func (bc *Blockchain) init() error {
genesisBlock, err := createGenesisBlock(bc.config)
if err != nil {

7
pkg/core/blockchain_test.go
View file

@ -174,8 +174,11 @@ func getTestBlockchain(t *testing.T) *Blockchain {
require.NoError(t, err, "could not create levelDB chain")
// adjust datadirectory to point to the correct folder
cfg.ApplicationConfiguration.DataDirectoryPath = "../rpc/chains/unit_testnet"
chain, err := NewBlockchainLevelDB(context.Background(), cfg)
cfg.ApplicationConfiguration.DBConfiguration.LevelDBOptions.DataDirectoryPath = "../rpc/chains/unit_testnet"
store, err := storage.NewLevelDBStore(context.Background(),
cfg.ApplicationConfiguration.DBConfiguration.LevelDBOptions)
assert.Nil(t, err)
chain, err := NewBlockchain(context.Background(), store, cfg.ProtocolConfiguration)
require.NoErrorf(t, err, "could not create levelDB chain")
return chain

13
pkg/core/storage/leveldb_store.go
View file

@ -8,6 +8,11 @@ import (
"github.com/syndtr/goleveldb/leveldb/util"
)
// LevelDBOptions configuration for LevelDB.
type LevelDBOptions struct {
DataDirectoryPath string `yaml:"DataDirectoryPath"`
}
// LevelDBStore is the official storage implementation for storing and retrieving
// blockchain data.
type LevelDBStore struct {
@ -17,8 +22,10 @@ type LevelDBStore struct {
// NewLevelDBStore return a new LevelDBStore object that will
// initialize the database found at the given path.
func NewLevelDBStore(ctx context.Context, path string, opts *opt.Options) (*LevelDBStore, error) {
db, err := leveldb.OpenFile(path, opts)
func NewLevelDBStore(ctx context.Context, cfg LevelDBOptions) (*LevelDBStore, error) {
var opts *opt.Options = nil // should be exposed via LevelDBOptions if anything needed
db, err := leveldb.OpenFile(cfg.DataDirectoryPath, opts)
if err != nil {
return nil, err
}
@ -30,7 +37,7 @@ func NewLevelDBStore(ctx context.Context, path string, opts *opt.Options) (*Leve
}()
return &LevelDBStore{
path: path,
path: cfg.DataDirectoryPath,
db: db,
}, nil
}

21
pkg/core/storage/redis_store.go
View file

@ -6,6 +6,13 @@ import (
"github.com/go-redis/redis"
)
// RedisDBOptions configuration for RedisDB.
type RedisDBOptions struct {
Addr string `yaml:"Addr"`
Password string `yaml:"Password"`
DB int `yaml:"DB"`
}
// RedisStore holds the client and maybe later some more metadata.
type RedisStore struct {
client *redis.Client
@ -33,19 +40,17 @@ func NewRedisBatch() *RedisBatch {
}
}
// NewRedisStore returns an new initialized - ready to use RedisStore object
func NewRedisStore() (*RedisStore, error) {
// NewRedisStore returns an new initialized - ready to use RedisStore object.
func NewRedisStore(cfg RedisDBOptions) (*RedisStore, error) {
c := redis.NewClient(&redis.Options{
Addr: "localhost:6379",
Password: "",
DB: 0,
Addr: cfg.Addr,
Password: cfg.Password,
DB: cfg.DB,
})
if _, err := c.Ping().Result(); err != nil {
return nil, err
}
return &RedisStore{
client: c,
}, nil
return &RedisStore{client: c}, nil
}
// Batch implements the Store interface.

16
pkg/core/storage/store.go
View file

@ -1,6 +1,7 @@
package storage
import (
"context"
"encoding/binary"
"errors"
)
@ -72,3 +73,18 @@ func AppendPrefixInt(k KeyPrefix, n int) []byte {
binary.LittleEndian.PutUint32(b, uint32(n))
return AppendPrefix(k, b)
}
// NewStore creates storage with preselected in configuration database type.
func NewStore(context context.Context, cfg DBConfiguration) (Store, error) {
var store Store
var err error
switch cfg.Type {
case "leveldb":
store, err = NewLevelDBStore(context, cfg.LevelDBOptions)
case "inmemory":
store = NewMemoryStore()
case "redis":
store, err = NewRedisStore(cfg.RedisDBOptions)
}
return store, err
}

10
pkg/core/storage/store_config.go Normal file
View file

@ -0,0 +1,10 @@
package storage
type (
// DBConfiguration describes configuration for DB. Supported: 'levelDB', 'redisDB'.
DBConfiguration struct {
Type string `yaml:"Type"`
LevelDBOptions LevelDBOptions `yaml:"LevelDBOptions"`
RedisDBOptions RedisDBOptions `yaml:"RedisDBOptions"`
}
)

6
pkg/rpc/server_test.go
View file

@ -13,6 +13,7 @@ import (
"github.com/CityOfZion/neo-go/config"
"github.com/CityOfZion/neo-go/pkg/core"
"github.com/CityOfZion/neo-go/pkg/core/storage"
"github.com/CityOfZion/neo-go/pkg/network"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -245,7 +246,10 @@ func TestHandler(t *testing.T) {
cfg, err := config.Load(configPath, net)
require.NoError(t, err, "could not load config")
chain, err := core.NewBlockchainLevelDB(context.Background(), cfg)
store, err := storage.NewLevelDBStore(context.Background(),
cfg.ApplicationConfiguration.DBConfiguration.LevelDBOptions)
assert.Nil(t, err)
chain, err := core.NewBlockchain(context.Background(), store, cfg.ProtocolConfiguration)
require.NoError(t, err, "could not create levelDB chain")
serverConfig := network.NewServerConfig(cfg)