From dafd385f5743270545b278990af57b4dab6a316a Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Sat, 19 Oct 2019 23:58:45 +0300 Subject: [PATCH 1/5] cli: extend NewCommand() to NewCommands() So that each module could now return an array of commands. It's gonna be used in the future to extend them. --- cli/main.go | 10 ++++------ cli/server/server.go | 8 ++++---- cli/smartcontract/smart_contract.go | 8 ++++---- cli/vm/vm.go | 8 ++++---- cli/wallet/wallet.go | 8 ++++---- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/cli/main.go b/cli/main.go index e58aaf157..6e2817ac9 100644 --- a/cli/main.go +++ b/cli/main.go @@ -17,12 +17,10 @@ func main() { ctl.Version = config.Version ctl.Usage = "Official Go client for Neo" - ctl.Commands = []cli.Command{ - server.NewCommand(), - smartcontract.NewCommand(), - wallet.NewCommand(), - vm.NewCommand(), - } + ctl.Commands = append(ctl.Commands, server.NewCommands()...) + ctl.Commands = append(ctl.Commands, smartcontract.NewCommands()...) + ctl.Commands = append(ctl.Commands, wallet.NewCommands()...) + ctl.Commands = append(ctl.Commands, vm.NewCommands()...) if err := ctl.Run(os.Args); err != nil { panic(err) diff --git a/cli/server/server.go b/cli/server/server.go index 53eb68c43..fd60f7e3b 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -16,9 +16,9 @@ import ( "github.com/urfave/cli" ) -// NewCommand creates a new Node command. -func NewCommand() cli.Command { - return cli.Command{ +// NewCommands returns 'node' command. +func NewCommands() []cli.Command { + return []cli.Command{{ Name: "node", Usage: "start a NEO node", Action: startServer, @@ -29,7 +29,7 @@ func NewCommand() cli.Command { cli.BoolFlag{Name: "testnet, t"}, cli.BoolFlag{Name: "debug, d"}, }, - } + }} } func newGraceContext() context.Context { diff --git a/cli/smartcontract/smart_contract.go b/cli/smartcontract/smart_contract.go index 4cd604e59..c967e42e7 100644 --- a/cli/smartcontract/smart_contract.go +++ b/cli/smartcontract/smart_contract.go @@ -35,9 +35,9 @@ func Main(op string, args []interface{}) { }` ) -// NewCommand returns a new contract command. -func NewCommand() cli.Command { - return cli.Command{ +// NewCommands returns 'contract' command. +func NewCommands() []cli.Command { + return []cli.Command{{ Name: "contract", Usage: "compile - debug - deploy smart contracts", Subcommands: []cli.Command{ @@ -98,7 +98,7 @@ func NewCommand() cli.Command { }, }, }, - } + }} } // initSmartContract initializes a given directory with some boiler plate code. diff --git a/cli/vm/vm.go b/cli/vm/vm.go index 35a8f92c2..7b3ac447b 100644 --- a/cli/vm/vm.go +++ b/cli/vm/vm.go @@ -9,9 +9,9 @@ import ( "github.com/urfave/cli" ) -// NewCommand creates a new VM command. -func NewCommand() cli.Command { - return cli.Command{ +// NewCommands returns 'vm' command. +func NewCommands() []cli.Command { + return []cli.Command{{ Name: "vm", Usage: "start the virtual machine", Action: startVMPrompt, @@ -31,7 +31,7 @@ func NewCommand() cli.Command { }, }, }, - } + }} } func startVMPrompt(ctx *cli.Context) error { diff --git a/cli/wallet/wallet.go b/cli/wallet/wallet.go index ce500e727..3091d74ed 100644 --- a/cli/wallet/wallet.go +++ b/cli/wallet/wallet.go @@ -16,9 +16,9 @@ var ( errPhraseMismatch = errors.New("the entered pass-phrases do not match. Maybe you have misspelled them") ) -// NewCommand creates a new Wallet command. -func NewCommand() cli.Command { - return cli.Command{ +// NewCommands returns 'wallet' command. +func NewCommands() []cli.Command { + return []cli.Command{{ Name: "wallet", Usage: "create, open and manage a NEO wallet", Subcommands: []cli.Command{ @@ -49,7 +49,7 @@ func NewCommand() cli.Command { }, }, }, - } + }} } func openWallet(ctx *cli.Context) error { From b533dfceba492afedef984e633777cdc981bb7c2 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 21 Oct 2019 08:37:01 +0300 Subject: [PATCH 2/5] core: don't panic on init when there are less than 2000 hashes If you're to sync less than 2000 headers no batched header key-value is gonna be written into the DB and init() would panic because bc.headerList.Len() would return 0. Use genesis block as a target in this case. --- pkg/core/blockchain.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index a20a6a5f1..263606ee1 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -137,7 +137,16 @@ func (bc *Blockchain) init() error { // that with stored blocks. if currHeaderHeight > bc.storedHeaderCount { hash := currHeaderHash - targetHash := bc.headerList.Get(bc.headerList.Len() - 1) + var targetHash util.Uint256 + if bc.headerList.Len() > 0 { + targetHash = bc.headerList.Get(bc.headerList.Len() - 1) + } else { + genesisBlock, err := createGenesisBlock(bc.config) + if err != nil { + return err + } + targetHash = genesisBlock.Hash() + } headers := make([]*Header, 0) for hash != targetHash { From c885a69edbbbd19cfd43ec725d1909521ef5d788 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 21 Oct 2019 08:41:05 +0300 Subject: [PATCH 3/5] cli: add db dump/restore commands Dump given number of blocks (from the given offset) to file and restore them from it. Fixes #436. --- cli/server/server.go | 214 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 192 insertions(+), 22 deletions(-) diff --git a/cli/server/server.go b/cli/server/server.go index fd60f7e3b..db5a2a036 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -9,6 +9,7 @@ import ( "github.com/CityOfZion/neo-go/config" "github.com/CityOfZion/neo-go/pkg/core" "github.com/CityOfZion/neo-go/pkg/core/storage" + "github.com/CityOfZion/neo-go/pkg/io" "github.com/CityOfZion/neo-go/pkg/network" "github.com/CityOfZion/neo-go/pkg/rpc" "github.com/pkg/errors" @@ -18,18 +19,63 @@ import ( // NewCommands returns 'node' command. func NewCommands() []cli.Command { - return []cli.Command{{ - Name: "node", - Usage: "start a NEO node", - Action: startServer, - Flags: []cli.Flag{ - cli.StringFlag{Name: "config-path"}, - cli.BoolFlag{Name: "privnet, p"}, - cli.BoolFlag{Name: "mainnet, m"}, - cli.BoolFlag{Name: "testnet, t"}, - cli.BoolFlag{Name: "debug, d"}, + var cfgFlags = []cli.Flag{ + cli.StringFlag{Name: "config-path"}, + cli.BoolFlag{Name: "privnet, p"}, + cli.BoolFlag{Name: "mainnet, m"}, + cli.BoolFlag{Name: "testnet, t"}, + cli.BoolFlag{Name: "debug, d"}, + } + var cfgWithCountFlags = make([]cli.Flag, len(cfgFlags)) + copy(cfgWithCountFlags, cfgFlags) + cfgWithCountFlags = append(cfgWithCountFlags, + cli.UintFlag{ + Name: "count, c", + Usage: "number of blocks to be processed (default or 0: all chain)", }, - }} + cli.UintFlag{ + Name: "skip, s", + Usage: "number of blocks to skip (default: 0)", + }, + ) + var cfgCountOutFlags = make([]cli.Flag, len(cfgWithCountFlags)) + copy(cfgCountOutFlags, cfgWithCountFlags) + cfgCountOutFlags = append(cfgCountOutFlags, cli.StringFlag{ + Name: "out, o", + Usage: "Output file (stdout if not given)", + }) + var cfgCountInFlags = make([]cli.Flag, len(cfgWithCountFlags)) + copy(cfgCountInFlags, cfgWithCountFlags) + cfgCountInFlags = append(cfgCountInFlags, cli.StringFlag{ + Name: "in, i", + Usage: "Input file (stdin if not given)", + }) + return []cli.Command{ + { + Name: "node", + Usage: "start a NEO node", + Action: startServer, + Flags: cfgFlags, + }, + { + Name: "db", + Usage: "database manipulations", + Subcommands: []cli.Command{ + { + Name: "dump", + Usage: "dump blocks (starting with block #1) to the file", + Action: dumpDB, + Flags: cfgCountOutFlags, + }, + { + Name: "restore", + Usage: "restore blocks from the file", + Action: restoreDB, + Flags: cfgCountInFlags, + }, + }, + }, + } } func newGraceContext() context.Context { @@ -43,26 +89,154 @@ func newGraceContext() context.Context { return ctx } -func startServer(ctx *cli.Context) error { - net := config.ModePrivNet +// getConfigFromContext looks at path and mode flags in the given config and +// returns appropriate config. +func getConfigFromContext(ctx *cli.Context) (config.Config, error) { + var net = config.ModePrivNet if ctx.Bool("testnet") { net = config.ModeTestNet } if ctx.Bool("mainnet") { net = config.ModeMainNet } - - grace, cancel := context.WithCancel(newGraceContext()) - defer cancel() - configPath := "./config" if argCp := ctx.String("config-path"); argCp != "" { configPath = argCp } - cfg, err := config.Load(configPath, net) + return config.Load(configPath, net) +} + +// handleLoggingParams enables debugging output is that's requested by the user. +func handleLoggingParams(ctx *cli.Context) { + if ctx.Bool("debug") { + log.SetLevel(log.DebugLevel) + } +} + +func getCountAndSkipFromContext(ctx *cli.Context) (uint32, uint32) { + count := uint32(ctx.Uint("count")) + skip := uint32(ctx.Uint("skip")) + return count, skip +} + +func dumpDB(ctx *cli.Context) error { + cfg, err := getConfigFromContext(ctx) if err != nil { return cli.NewExitError(err, 1) } + handleLoggingParams(ctx) + count, skip := getCountAndSkipFromContext(ctx) + + var outStream = os.Stdout + if out := ctx.String("out"); out != "" { + outStream, err = os.Create(out) + if err != nil { + return cli.NewExitError(err, 1) + } + } + defer outStream.Close() + writer := io.NewBinWriterFromIO(outStream) + + grace, cancel := context.WithCancel(newGraceContext()) + defer cancel() + + chain, err := initBlockChain(cfg) + if err != nil { + return cli.NewExitError(err, 1) + } + go chain.Run(grace) + + chainHeight := chain.BlockHeight() + if skip+count > chainHeight { + return cli.NewExitError(fmt.Errorf("chain is not that high (%d) to dump %d blocks starting from %d", chainHeight, count, skip), 1) + } + if count == 0 { + count = chainHeight - skip + } + writer.WriteLE(count) + for i := skip + 1; i <= count; i++ { + bh := chain.GetHeaderHash(int(i)) + b, err := chain.GetBlock(bh) + if err != nil { + return cli.NewExitError(fmt.Errorf("failed to get block %d: %s", i, err), 1) + } + b.EncodeBinary(writer) + if writer.Err != nil { + return cli.NewExitError(err, 1) + } + } + return nil +} +func restoreDB(ctx *cli.Context) error { + cfg, err := getConfigFromContext(ctx) + if err != nil { + return err + } + handleLoggingParams(ctx) + count, skip := getCountAndSkipFromContext(ctx) + + var inStream = os.Stdin + if in := ctx.String("in"); in != "" { + inStream, err = os.Open(in) + if err != nil { + return cli.NewExitError(err, 1) + } + } + defer inStream.Close() + reader := io.NewBinReaderFromIO(inStream) + + grace, cancel := context.WithCancel(newGraceContext()) + defer cancel() + + chain, err := initBlockChain(cfg) + if err != nil { + return err + } + go chain.Run(grace) + + var allBlocks uint32 + reader.ReadLE(&allBlocks) + if reader.Err != nil { + return cli.NewExitError(err, 1) + } + if skip+count > allBlocks { + return cli.NewExitError(fmt.Errorf("input file has only %d blocks, can't read %d starting from %d", allBlocks, count, skip), 1) + } + if count == 0 { + count = allBlocks + } + i := uint32(0) + for ; i < skip; i++ { + b := &core.Block{} + b.DecodeBinary(reader) + if reader.Err != nil { + return cli.NewExitError(err, 1) + } + } + for ; i < count; i++ { + b := &core.Block{} + b.DecodeBinary(reader) + if reader.Err != nil { + return cli.NewExitError(err, 1) + } + err := chain.AddBlock(b) + if err != nil { + return cli.NewExitError(fmt.Errorf("failed to add block %d: %s", i, err), 1) + } + } + + return nil +} + +func startServer(ctx *cli.Context) error { + cfg, err := getConfigFromContext(ctx) + if err != nil { + return err + } + handleLoggingParams(ctx) + + grace, cancel := context.WithCancel(newGraceContext()) + defer cancel() serverConfig := network.NewServerConfig(cfg) @@ -71,10 +245,6 @@ func startServer(ctx *cli.Context) error { return err } - if ctx.Bool("debug") { - log.SetLevel(log.DebugLevel) - } - server := network.NewServer(serverConfig, chain) rpcServer := rpc.NewServer(chain, cfg.ApplicationConfiguration.RPCPort, server) errChan := make(chan error) From 70407f0c19a743f05375dc05cf7aa5436356ce06 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 21 Oct 2019 10:04:58 +0300 Subject: [PATCH 4/5] core: remove unused context parameter from persist() --- pkg/core/blockchain.go | 6 +++--- pkg/core/blockchain_test.go | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 263606ee1..2946633ff 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -175,7 +175,7 @@ func (bc *Blockchain) Run(ctx context.Context) { persistTimer := time.NewTimer(persistInterval) defer func() { persistTimer.Stop() - if err := bc.persist(ctx); err != nil { + if err := bc.persist(); err != nil { log.Warnf("failed to persist: %s", err) } if err := bc.store.Close(); err != nil { @@ -191,7 +191,7 @@ func (bc *Blockchain) Run(ctx context.Context) { bc.headersOpDone <- struct{}{} case <-persistTimer.C: go func() { - err := bc.persist(ctx) + err := bc.persist() if err != nil { log.Warnf("failed to persist blockchain: %s", err) } @@ -470,7 +470,7 @@ func (bc *Blockchain) storeBlock(block *Block) error { } // persist flushes current in-memory store contents to the persistent storage. -func (bc *Blockchain) persist(ctx context.Context) error { +func (bc *Blockchain) persist() error { var ( start = time.Now() persisted int diff --git a/pkg/core/blockchain_test.go b/pkg/core/blockchain_test.go index 71756941d..9e3b5a515 100644 --- a/pkg/core/blockchain_test.go +++ b/pkg/core/blockchain_test.go @@ -1,7 +1,6 @@ package core import ( - "context" "testing" "github.com/CityOfZion/neo-go/pkg/core/storage" @@ -53,7 +52,7 @@ func TestAddBlock(t *testing.T) { assert.Equal(t, lastBlock.Hash(), bc.CurrentHeaderHash()) // This one tests persisting blocks, so it does need to persist() - require.NoError(t, bc.persist(context.Background())) + require.NoError(t, bc.persist()) for _, block := range blocks { key := storage.AppendPrefix(storage.DataBlock, block.Hash().BytesReverse()) @@ -82,7 +81,7 @@ func TestGetHeader(t *testing.T) { b2 := newBlock(2) _, err = bc.GetHeader(b2.Hash()) assert.Error(t, err) - assert.NoError(t, bc.persist(context.Background())) + assert.NoError(t, bc.persist()) } } @@ -106,7 +105,7 @@ func TestGetBlock(t *testing.T) { assert.Equal(t, blocks[i].Index, block.Index) assert.Equal(t, blocks[i].Hash(), block.Hash()) } - assert.NoError(t, bc.persist(context.Background())) + assert.NoError(t, bc.persist()) } } @@ -127,7 +126,7 @@ func TestHasBlock(t *testing.T) { } newBlock := newBlock(51) assert.False(t, bc.HasBlock(newBlock.Hash())) - assert.NoError(t, bc.persist(context.Background())) + assert.NoError(t, bc.persist()) } } @@ -151,6 +150,6 @@ func TestGetTransaction(t *testing.T) { assert.Equal(t, 1, io.GetVarSize(tx.Inputs)) assert.Equal(t, 1, io.GetVarSize(tx.Outputs)) assert.Equal(t, 1, io.GetVarSize(tx.Scripts)) - assert.NoError(t, bc.persist(context.Background())) + assert.NoError(t, bc.persist()) } } From 07c2105aa5b05d49b62454837f19ec33ed56032f Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 21 Oct 2019 11:05:10 +0300 Subject: [PATCH 5/5] core: log values from the store in persist() We're about stored values here, so print those, which avoids blocking in bc.HeaderHeight() and removes duplication between blockHeight and persistedHeight. Fixes saving the blockchain on exit (deferred function in Run() blocked in persist()). Test modification was required because storeBlocks() doesn't actually save headers and thus TestGetTransaction started to fail on persist(). --- pkg/core/blockchain.go | 9 ++++++--- pkg/core/blockchain_test.go | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 2946633ff..3cbf95afa 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -488,13 +488,16 @@ func (bc *Blockchain) persist() error { oldHeight := atomic.SwapUint32(&bc.persistedHeight, bHeight) diff := bHeight - oldHeight + storedHeaderHeight, _, err := storage.CurrentHeaderHeight(bc.store) + if err != nil { + return err + } if persisted > 0 { log.WithFields(log.Fields{ "persistedBlocks": diff, "persistedKeys": persisted, - "headerHeight": bc.HeaderHeight(), - "blockHeight": bc.BlockHeight(), - "persistedHeight": atomic.LoadUint32(&bc.persistedHeight), + "headerHeight": storedHeaderHeight, + "blockHeight": bHeight, "took": time.Since(start), }).Info("blockchain persist completed") } diff --git a/pkg/core/blockchain_test.go b/pkg/core/blockchain_test.go index 9e3b5a515..f99114893 100644 --- a/pkg/core/blockchain_test.go +++ b/pkg/core/blockchain_test.go @@ -134,10 +134,12 @@ func TestGetTransaction(t *testing.T) { b1 := getDecodedBlock(t, 1) block := getDecodedBlock(t, 2) bc := newTestChain(t) + // Turn verification off, because these blocks are really from some other chain + // and can't be verified, but we don't care about that in this test. + bc.config.VerifyBlocks = false - // These are from some kind of different chain, so can't be added via AddBlock(). - assert.Nil(t, bc.storeBlock(b1)) - assert.Nil(t, bc.storeBlock(block)) + assert.Nil(t, bc.AddBlock(b1)) + assert.Nil(t, bc.AddBlock(block)) // Test unpersisted and persisted access for j := 0; j < 2; j++ {