From 7aa7490fb256d0469d7ed6d836ae33d6f1b68800 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 26 Dec 2019 20:03:06 +0300 Subject: [PATCH 1/5] cli: fix wrong db dump format It was broken by 03ff2976ed6b6fa0a77646dcedbe5dd71c587860 that changed fixed 32-bit length encoding to VarBytes. --- cli/server/server.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/server/server.go b/cli/server/server.go index 965e3fb2a..314f08217 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -193,7 +193,8 @@ func dumpDB(ctx *cli.Context) error { buf := io.NewBufBinWriter() b.EncodeBinary(buf.BinWriter) bytes := buf.Bytes() - writer.WriteVarBytes(bytes) + writer.WriteU32LE(uint32(len(bytes))) + writer.WriteBytes(bytes) if writer.Err != nil { return cli.NewExitError(err, 1) } From b7061301756231d241f481f09808eccf7190b51d Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 27 Dec 2019 12:11:57 +0300 Subject: [PATCH 2/5] cli/server: redo dumper to dump blocks from 0 NGD dumps are all zero-based and even though I don't like it (genesys block should not be imported, it's the root of chain trust), we have to conform to this convention for interoperability with C# nodes (otherwise they're not able to import our dumps). This also renames `skip` dumper parameter to `start` which is more logical now, the default is to start the dump from block number zero. --- cli/server/server.go | 54 ++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/cli/server/server.go b/cli/server/server.go index 314f08217..876d4b4aa 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -35,23 +35,31 @@ func NewCommands() []cli.Command { Name: "count, c", Usage: "number of blocks to be processed (default or 0: all chain)", }, + ) + var cfgCountOutFlags = make([]cli.Flag, len(cfgWithCountFlags)) + copy(cfgCountOutFlags, cfgWithCountFlags) + cfgCountOutFlags = append(cfgCountOutFlags, + cli.UintFlag{ + Name: "start, s", + Usage: "block number to start from (default: 0)", + }, + cli.StringFlag{ + Name: "out, o", + Usage: "Output file (stdout if not given)", + }, + ) + var cfgCountInFlags = make([]cli.Flag, len(cfgWithCountFlags)) + copy(cfgCountInFlags, cfgWithCountFlags) + cfgCountInFlags = append(cfgCountInFlags, cli.UintFlag{ Name: "skip, s", Usage: "number of blocks to skip (default: 0)", }, + cli.StringFlag{ + Name: "in, i", + Usage: "Input file (stdin if not given)", + }, ) - var cfgCountOutFlags = make([]cli.Flag, len(cfgWithCountFlags)) - copy(cfgCountOutFlags, cfgWithCountFlags) - cfgCountOutFlags = append(cfgCountOutFlags, cli.StringFlag{ - Name: "out, o", - Usage: "Output file (stdout if not given)", - }) - var cfgCountInFlags = make([]cli.Flag, len(cfgWithCountFlags)) - copy(cfgCountInFlags, cfgWithCountFlags) - cfgCountInFlags = append(cfgCountInFlags, cli.StringFlag{ - Name: "in, i", - Usage: "Input file (stdin if not given)", - }) return []cli.Command{ { Name: "node", @@ -129,12 +137,6 @@ func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) return nil } -func getCountAndSkipFromContext(ctx *cli.Context) (uint32, uint32) { - count := uint32(ctx.Uint("count")) - skip := uint32(ctx.Uint("skip")) - return count, skip -} - func initBCWithMetrics(cfg config.Config) (*core.Blockchain, *metrics.Service, *metrics.Service, error) { chain, err := initBlockChain(cfg) if err != nil { @@ -159,7 +161,8 @@ func dumpDB(ctx *cli.Context) error { if err := handleLoggingParams(ctx, cfg.ApplicationConfiguration); err != nil { return cli.NewExitError(err, 1) } - count, skip := getCountAndSkipFromContext(ctx) + count := uint32(ctx.Uint("count")) + start := uint32(ctx.Uint("start")) var outStream = os.Stdout if out := ctx.String("out"); out != "" { @@ -176,15 +179,15 @@ func dumpDB(ctx *cli.Context) error { return err } - chainHeight := chain.BlockHeight() - if skip+count > chainHeight { - return cli.NewExitError(fmt.Errorf("chain is not that high (%d) to dump %d blocks starting from %d", chainHeight, count, skip), 1) + chainCount := chain.BlockHeight() + 1 + if start+count > chainCount { + return cli.NewExitError(fmt.Errorf("chain is not that high (%d) to dump %d blocks starting from %d", chainCount-1, count, start), 1) } if count == 0 { - count = chainHeight - skip + count = chainCount - start } writer.WriteU32LE(count) - for i := skip + 1; i <= skip+count; i++ { + for i := start; i < start+count; i++ { bh := chain.GetHeaderHash(int(i)) b, err := chain.GetBlock(bh) if err != nil { @@ -213,7 +216,8 @@ func restoreDB(ctx *cli.Context) error { if err := handleLoggingParams(ctx, cfg.ApplicationConfiguration); err != nil { return cli.NewExitError(err, 1) } - count, skip := getCountAndSkipFromContext(ctx) + count := uint32(ctx.Uint("count")) + skip := uint32(ctx.Uint("skip")) var inStream = os.Stdin if in := ctx.String("in"); in != "" { From ae003a1578840c12378d72c2781a7f4c0a57c7d3 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 27 Dec 2019 12:15:47 +0300 Subject: [PATCH 3/5] cli/server: fix db restorer math wrt skip Given `-s 1` with a dump of 6001 blocks it skipped the first one and then tried to import the next 6001 which failed with EOF because there are only 6000 blocks left. --- cli/server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/server/server.go b/cli/server/server.go index 876d4b4aa..ae2288b01 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -242,7 +242,7 @@ func restoreDB(ctx *cli.Context) error { return cli.NewExitError(fmt.Errorf("input file has only %d blocks, can't read %d starting from %d", allBlocks, count, skip), 1) } if count == 0 { - count = allBlocks + count = allBlocks - skip } i := uint32(0) for ; i < skip; i++ { From 1f4b7b366efb4aa300d7fc1510fff4d793c8452f Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 27 Dec 2019 12:25:39 +0300 Subject: [PATCH 4/5] cli/server: skip genesis block on restore if it matches ours Enables more convenient imports without skipping over the block 0. --- cli/server/server.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cli/server/server.go b/cli/server/server.go index ae2288b01..9f994827d 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -259,6 +259,13 @@ func restoreDB(ctx *cli.Context) error { if err != nil { return cli.NewExitError(err, 1) } + if block.Index == 0 && i == 0 && skip == 0 { + genesis, err := chain.GetBlock(block.Hash()) + if err == nil && genesis.Index == 0 { + log.Info("skipped genesis block ", block.Hash().StringLE()) + continue + } + } err = chain.AddBlock(block) if err != nil { return cli.NewExitError(fmt.Errorf("failed to add block %d: %s", i, err), 1) From f15a2401b1cf9dbb30fc9819dc7b65236fa15c39 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 27 Dec 2019 12:38:07 +0300 Subject: [PATCH 5/5] cli/server: close the chain gracefully on restore even on error Not an issue for dumper, but when restoring we should correctly save everything already imported even if the subsequent block fails. --- cli/server/server.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/server/server.go b/cli/server/server.go index 9f994827d..6ecdfdda7 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -233,6 +233,9 @@ func restoreDB(ctx *cli.Context) error { if err != nil { return err } + defer chain.Close() + defer prometheus.ShutDown() + defer pprof.ShutDown() var allBlocks = reader.ReadU32LE() if reader.Err != nil { @@ -271,9 +274,6 @@ func restoreDB(ctx *cli.Context) error { return cli.NewExitError(fmt.Errorf("failed to add block %d: %s", i, err), 1) } } - pprof.ShutDown() - prometheus.ShutDown() - chain.Close() return nil }