From 176c0e98d6eba33d9dfc0383c34820c2e5f75575 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 24 Jun 2020 16:09:54 +0300 Subject: [PATCH 1/2] cli/server: dump genesis block state when restoring Neo 3 node does it too and it's very useful because now the genesis block also does some state changes that are important. --- cli/server/dump.go | 8 ++++---- cli/server/server.go | 15 +++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cli/server/dump.go b/cli/server/dump.go index e6084a9ee..303c7e903 100644 --- a/cli/server/dump.go +++ b/cli/server/dump.go @@ -165,8 +165,8 @@ func readFile(path string) (*dump, error) { // File dump-block-$FILENO.json contains blocks from $FILENO-999, $FILENO // Example: file `BlockStorage_100000/dump-block-6000.json` contains blocks from 5001 to 6000. func getPath(prefix string, index uint32) (string, error) { - dirN := (index-1)/100000 + 1 - dir := fmt.Sprintf("BlockStorage_%d00000", dirN) + dirN := ((index + 99999) / 100000) * 100000 + dir := fmt.Sprintf("BlockStorage_%d", dirN) path := filepath.Join(prefix, dir) info, err := os.Stat(path) @@ -179,7 +179,7 @@ func getPath(prefix string, index uint32) (string, error) { return "", fmt.Errorf("file `%s` is not a directory", path) } - fileN := (index-1)/1000 + 1 - file := fmt.Sprintf("dump-block-%d000.json", fileN) + fileN := ((index + 999) / 1000) * 1000 + file := fmt.Sprintf("dump-block-%d.json", fileN) return filepath.Join(path, file), nil } diff --git a/cli/server/server.go b/cli/server/server.go index be46edd2d..63b494655 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -292,16 +292,19 @@ func restoreDB(ctx *cli.Context) error { genesis, err := chain.GetBlock(block.Hash()) if err == nil && genesis.Index == 0 { log.Info("skipped genesis block", zap.String("hash", block.Hash().StringLE())) - continue + } + } else { + err = chain.AddBlock(block) + if err != nil { + return cli.NewExitError(fmt.Errorf("failed to add block %d: %s", i, err), 1) } } - err = chain.AddBlock(block) - if err != nil { - return cli.NewExitError(fmt.Errorf("failed to add block %d: %s", i, err), 1) - } - if dumpDir != "" { batch := chain.LastBatch() + // The genesis block may already be persisted, so LastBatch() will return nil. + if batch == nil && block.Index == 0 { + continue + } dump.add(block.Index, batch) lastIndex = block.Index if block.Index%1000 == 0 { From b0c063aaaf7ccfc81ba793cbf54bc1ccb3372fae Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 24 Jun 2020 16:11:34 +0300 Subject: [PATCH 2/2] scripts: replace compare-dumps with go version It's a bit faster and it's also updated to handle block 0. Usage: $ go run scripts/compare-dump.go a b With the same functionality as the old script. --- scripts/compare-dumps | 63 --------------- scripts/compare-dumps.go | 167 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 63 deletions(-) delete mode 100755 scripts/compare-dumps create mode 100644 scripts/compare-dumps.go diff --git a/scripts/compare-dumps b/scripts/compare-dumps deleted file mode 100755 index da41dcd11..000000000 --- a/scripts/compare-dumps +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh - -ARG1=$1 -ARG2=$2 -if [ -z "$ARG1" ] || [ -z "$ARG2" ]; then - echo one of the arguments is empty - exit 1 -fi - - -compare() { - # replace replaces storage operation from "Changed" to "Added" - # normalize replaces performs replace and sorts keys in lexicographic order - # next we normalize every json file - # and finally compare them as a whole - jq --argfile x "$1" --argfile y "$2" \ - -n 'def replace: map(if (.state == "Changed") then (.state="Added") else . end); - def normalize: .storage = (.storage | replace | sort_by(.key)); - ($x | map(normalize)) as $x - | ($y | map(normalize)) as $y - | $x | range(length) | . as $i | select($x[$i] != $y[$i]) | $x[$i].block | halt_error(1)' -} - -if [ -f "$ARG1" ] && [ -f "$ARG2" ]; then - compare "$ARG1" "$ARG2" - if [ $? -ne 0 ]; then - echo failed - exit 1 - fi - - exit 0 -fi - -if [ ! -d "$ARG1" ] || [ ! -d "$ARG2" ]; then - echo both arguments must have the same type and exist - exit 1 -fi - -FIRST=$3 -if [ -z "$FIRST" ]; then - FIRST=1 -fi - -LAST=$4 -if [ -z "$LAST" ]; then - LAST=40 # 40_00000 block -fi - -# directories contain 100k blocks -for i in `seq $FIRST $LAST`; do - dir=BlockStorage_${i}00000 - echo Processing directory $dir - - # files are grouped by 1k blocks - for j in `seq $(((i-1)*100 + 1)) $((i*100))`; do - file=dump-block-${j}000.json - compare "$ARG1/$dir/$file" "$ARG2/$dir/$file" - if [ $? -ne 0 ]; then - echo failed on file $dir/$file - exit 1 - fi - done -done diff --git a/scripts/compare-dumps.go b/scripts/compare-dumps.go new file mode 100644 index 000000000..268806e6d --- /dev/null +++ b/scripts/compare-dumps.go @@ -0,0 +1,167 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + + "github.com/urfave/cli" +) + +type dump []blockDump + +type blockDump struct { + Block uint32 `json:"block"` + Size int `json:"size"` + Storage []storageOp `json:"storage"` +} + +type storageOp struct { + State string `json:"state"` + Key string `json:"key"` + Value string `json:"value,omitempty"` +} + +func readFile(path string) (dump, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + d := make(dump, 0) + if err := json.Unmarshal(data, &d); err != nil { + return nil, err + } + return d, err +} + +func (d dump) normalize() { + for i := range d { + for j := range d[i].Storage { + if d[i].Storage[j].State == "Changed" { + d[i].Storage[j].State = "Added" + } + } + sort.Slice(d[i].Storage, func(k, l int) bool { + return d[i].Storage[k].Key < d[i].Storage[l].Key + }) + } + // assume that d is already sorted by Block +} + +func compare(a, b string) error { + dumpA, err := readFile(a) + if err != nil { + return fmt.Errorf("reading file %s: %v", a, err) + } + dumpB, err := readFile(b) + if err != nil { + return fmt.Errorf("reading file %s: %v", b, err) + } + dumpA.normalize() + dumpB.normalize() + if len(dumpA) != len(dumpB) { + return fmt.Errorf("dump files differ in size: %d vs %d", len(dumpA), len(dumpB)) + } + for i := range dumpA { + blockA := &dumpA[i] + blockB := &dumpB[i] + if blockA.Block != blockB.Block { + return fmt.Errorf("block number mismatch: %d vs %d", blockA.Block, blockB.Block) + } + if blockA.Size != blockB.Size { + return fmt.Errorf("block %d, changes number mismatch: %d vs %d", blockA.Block, blockA.Size, blockB.Size) + } + if len(blockA.Storage) != len(blockB.Storage) { + return fmt.Errorf("block %d, changes length mismatch: %d vs %d", blockA.Block, len(blockA.Storage), len(blockB.Storage)) + } + fail := false + for j := range blockA.Storage { + if blockA.Storage[j].Key != blockB.Storage[j].Key { + return fmt.Errorf("block %d: key mismatch: %s vs %s", blockA.Block, blockA.Storage[j].Key, blockB.Storage[j].Key) + } + if blockA.Storage[j].State != blockB.Storage[j].State { + return fmt.Errorf("block %d: state mismatch for key %s: %s vs %s", blockA.Block, blockA.Storage[j].Key, blockA.Storage[j].State, blockB.Storage[j].State) + } + if blockA.Storage[j].Value != blockB.Storage[j].Value { + fail = true + fmt.Printf("block %d: value mismatch for key %s: %s vs %s\n", blockA.Block, blockA.Storage[j].Key, blockA.Storage[j].Value, blockB.Storage[j].Value) + } + } + if fail { + return errors.New("fail") + } + } + return nil +} + +func cliMain(c *cli.Context) error { + a := c.Args().Get(0) + b := c.Args().Get(1) + if a == "" { + return errors.New("no arguments given") + } + if b == "" { + return errors.New("missing second argument") + } + fa, err := os.Open(a) + if err != nil { + return err + } + defer fa.Close() + fb, err := os.Open(b) + if err != nil { + return err + } + defer fb.Close() + + astat, err := fa.Stat() + if err != nil { + return err + } + bstat, err := fb.Stat() + if err != nil { + return err + } + if astat.Mode().IsRegular() && bstat.Mode().IsRegular() { + return compare(a, b) + } + if astat.Mode().IsDir() && bstat.Mode().IsDir() { + for i := 0; i <= 6000000; i += 100000 { + dir := fmt.Sprintf("BlockStorage_%d", i) + fmt.Println("Processing directory", dir) + for j := i - 99000; j <= i; j += 1000 { + if j < 0 { + continue + } + fname := fmt.Sprintf("%s/dump-block-%d.json", dir, j) + + aname := filepath.Join(a, fname) + bname := filepath.Join(b, fname) + err := compare(aname, bname) + if err != nil { + return fmt.Errorf("file %s: %v", fname, err) + } + } + } + return nil + } + return errors.New("both parameters must be either dump files or directories") +} + +func main() { + ctl := cli.NewApp() + ctl.Name = "compare-dumps" + ctl.Version = "1.0" + ctl.Usage = "compare-dumps dumpDirA dumpDirB" + ctl.Action = cliMain + + if err := ctl.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + fmt.Fprintln(os.Stderr, ctl.Usage) + os.Exit(1) + } +}