Merge pull request #1098 from nspcc-dev/dump-block-0

Dump block 0 changes

cli/server/dump.go

@@ -165,8 +165,8 @@ func readFile(path string) (*dump, error) {
 // File dump-block-$FILENO.json contains blocks from $FILENO-999, $FILENO
 // Example: file `BlockStorage_100000/dump-block-6000.json` contains blocks from 5001 to 6000.
 func getPath(prefix string, index uint32) (string, error) {
-	dirN := (index-1)/100000 + 1
-	dir := fmt.Sprintf("BlockStorage_%d00000", dirN)
+	dirN := ((index + 99999) / 100000) * 100000
+	dir := fmt.Sprintf("BlockStorage_%d", dirN)
 
 	path := filepath.Join(prefix, dir)
 	info, err := os.Stat(path)
@@ -179,7 +179,7 @@ func getPath(prefix string, index uint32) (string, error) {
 		return "", fmt.Errorf("file `%s` is not a directory", path)
 	}
 
-	fileN := (index-1)/1000 + 1
-	file := fmt.Sprintf("dump-block-%d000.json", fileN)
+	fileN := ((index + 999) / 1000) * 1000
+	file := fmt.Sprintf("dump-block-%d.json", fileN)
 	return filepath.Join(path, file), nil
 }

cli/server/server.go

@@ -292,16 +292,19 @@ func restoreDB(ctx *cli.Context) error {
 			genesis, err := chain.GetBlock(block.Hash())
 			if err == nil && genesis.Index == 0 {
 				log.Info("skipped genesis block", zap.String("hash", block.Hash().StringLE()))
-				continue
+			}
+		} else {
+			err = chain.AddBlock(block)
+			if err != nil {
+				return cli.NewExitError(fmt.Errorf("failed to add block %d: %s", i, err), 1)
 			}
 		}
-		err = chain.AddBlock(block)
-		if err != nil {
-			return cli.NewExitError(fmt.Errorf("failed to add block %d: %s", i, err), 1)
-		}
-
 		if dumpDir != "" {
 			batch := chain.LastBatch()
+			// The genesis block may already be persisted, so LastBatch() will return nil.
+			if batch == nil && block.Index == 0 {
+				continue
+			}
 			dump.add(block.Index, batch)
 			lastIndex = block.Index
 			if block.Index%1000 == 0 {

scripts/compare-dumps

@@ -1,63 +0,0 @@
-#!/bin/sh
-
-ARG1=$1
-ARG2=$2
-if [ -z "$ARG1" ] || [ -z "$ARG2" ]; then
-	echo one of the arguments is empty
-	exit 1
-fi
-
-
-compare() {
-	# replace replaces storage operation from "Changed" to "Added"
-	# normalize replaces performs replace and sorts keys in lexicographic order
-	# next we normalize every json file
-	# and finally compare them as a whole
-	jq --argfile x "$1" --argfile y "$2" \
-      -n 'def replace: map(if (.state == "Changed") then (.state="Added") else . end);
-          def normalize: .storage = (.storage | replace | sort_by(.key));
-          ($x | map(normalize)) as $x
-        | ($y | map(normalize)) as $y
-        | $x | range(length) | . as $i | select($x[$i] != $y[$i]) | $x[$i].block | halt_error(1)'
-}
-
-if [ -f "$ARG1" ] && [ -f "$ARG2" ]; then
-	compare "$ARG1" "$ARG2"
-	if [ $? -ne 0 ]; then
-		echo failed
-		exit 1
-	fi
-
-    exit 0
-fi
-
-if [ ! -d "$ARG1" ] || [ ! -d "$ARG2" ]; then
-	echo both arguments must have the same type and exist
-	exit 1
-fi
-
-FIRST=$3
-if [ -z "$FIRST" ]; then
-    FIRST=1
-fi
-
-LAST=$4
-if [ -z "$LAST" ]; then
-    LAST=40 # 40_00000 block
-fi
-
-# directories contain 100k blocks
-for i in `seq $FIRST $LAST`; do
-	dir=BlockStorage_${i}00000
-	echo Processing directory $dir
-
-	# files are grouped by 1k blocks
-	for j in `seq $(((i-1)*100 + 1)) $((i*100))`; do
-		file=dump-block-${j}000.json
-		compare "$ARG1/$dir/$file" "$ARG2/$dir/$file"
-		if [ $? -ne 0 ]; then
-			echo failed on file $dir/$file
-			exit 1
-		fi
-	done
-done

scripts/compare-dumps.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/urfave/cli"
+)
+
+type dump []blockDump
+
+type blockDump struct {
+	Block   uint32      `json:"block"`
+	Size    int         `json:"size"`
+	Storage []storageOp `json:"storage"`
+}
+
+type storageOp struct {
+	State string `json:"state"`
+	Key   string `json:"key"`
+	Value string `json:"value,omitempty"`
+}
+
+func readFile(path string) (dump, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	d := make(dump, 0)
+	if err := json.Unmarshal(data, &d); err != nil {
+		return nil, err
+	}
+	return d, err
+}
+
+func (d dump) normalize() {
+	for i := range d {
+		for j := range d[i].Storage {
+			if d[i].Storage[j].State == "Changed" {
+				d[i].Storage[j].State = "Added"
+			}
+		}
+		sort.Slice(d[i].Storage, func(k, l int) bool {
+			return d[i].Storage[k].Key < d[i].Storage[l].Key
+		})
+	}
+	// assume that d is already sorted by Block
+}
+
+func compare(a, b string) error {
+	dumpA, err := readFile(a)
+	if err != nil {
+		return fmt.Errorf("reading file %s: %v", a, err)
+	}
+	dumpB, err := readFile(b)
+	if err != nil {
+		return fmt.Errorf("reading file %s: %v", b, err)
+	}
+	dumpA.normalize()
+	dumpB.normalize()
+	if len(dumpA) != len(dumpB) {
+		return fmt.Errorf("dump files differ in size: %d vs %d", len(dumpA), len(dumpB))
+	}
+	for i := range dumpA {
+		blockA := &dumpA[i]
+		blockB := &dumpB[i]
+		if blockA.Block != blockB.Block {
+			return fmt.Errorf("block number mismatch: %d vs %d", blockA.Block, blockB.Block)
+		}
+		if blockA.Size != blockB.Size {
+			return fmt.Errorf("block %d, changes number mismatch: %d vs %d", blockA.Block, blockA.Size, blockB.Size)
+		}
+		if len(blockA.Storage) != len(blockB.Storage) {
+			return fmt.Errorf("block %d, changes length mismatch: %d vs %d", blockA.Block, len(blockA.Storage), len(blockB.Storage))
+		}
+		fail := false
+		for j := range blockA.Storage {
+			if blockA.Storage[j].Key != blockB.Storage[j].Key {
+				return fmt.Errorf("block %d: key mismatch: %s vs %s", blockA.Block, blockA.Storage[j].Key, blockB.Storage[j].Key)
+			}
+			if blockA.Storage[j].State != blockB.Storage[j].State {
+				return fmt.Errorf("block %d: state mismatch for key %s: %s vs %s", blockA.Block, blockA.Storage[j].Key, blockA.Storage[j].State, blockB.Storage[j].State)
+			}
+			if blockA.Storage[j].Value != blockB.Storage[j].Value {
+				fail = true
+				fmt.Printf("block %d: value mismatch for key %s: %s vs %s\n", blockA.Block, blockA.Storage[j].Key, blockA.Storage[j].Value, blockB.Storage[j].Value)
+			}
+		}
+		if fail {
+			return errors.New("fail")
+		}
+	}
+	return nil
+}
+
+func cliMain(c *cli.Context) error {
+	a := c.Args().Get(0)
+	b := c.Args().Get(1)
+	if a == "" {
+		return errors.New("no arguments given")
+	}
+	if b == "" {
+		return errors.New("missing second argument")
+	}
+	fa, err := os.Open(a)
+	if err != nil {
+		return err
+	}
+	defer fa.Close()
+	fb, err := os.Open(b)
+	if err != nil {
+		return err
+	}
+	defer fb.Close()
+
+	astat, err := fa.Stat()
+	if err != nil {
+		return err
+	}
+	bstat, err := fb.Stat()
+	if err != nil {
+		return err
+	}
+	if astat.Mode().IsRegular() && bstat.Mode().IsRegular() {
+		return compare(a, b)
+	}
+	if astat.Mode().IsDir() && bstat.Mode().IsDir() {
+		for i := 0; i <= 6000000; i += 100000 {
+			dir := fmt.Sprintf("BlockStorage_%d", i)
+			fmt.Println("Processing directory", dir)
+			for j := i - 99000; j <= i; j += 1000 {
+				if j < 0 {
+					continue
+				}
+				fname := fmt.Sprintf("%s/dump-block-%d.json", dir, j)
+
+				aname := filepath.Join(a, fname)
+				bname := filepath.Join(b, fname)
+				err := compare(aname, bname)
+				if err != nil {
+					return fmt.Errorf("file %s: %v", fname, err)
+				}
+			}
+		}
+		return nil
+	}
+	return errors.New("both parameters must be either dump files or directories")
+}
+
+func main() {
+	ctl := cli.NewApp()
+	ctl.Name = "compare-dumps"
+	ctl.Version = "1.0"
+	ctl.Usage = "compare-dumps dumpDirA dumpDirB"
+	ctl.Action = cliMain
+
+	if err := ctl.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		fmt.Fprintln(os.Stderr, ctl.Usage)
+		os.Exit(1)
+	}
+}