diff --git a/scripts/compare-dumps b/scripts/compare-dumps deleted file mode 100755 index da41dcd11..000000000 --- a/scripts/compare-dumps +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh - -ARG1=$1 -ARG2=$2 -if [ -z "$ARG1" ] || [ -z "$ARG2" ]; then - echo one of the arguments is empty - exit 1 -fi - - -compare() { - # replace replaces storage operation from "Changed" to "Added" - # normalize replaces performs replace and sorts keys in lexicographic order - # next we normalize every json file - # and finally compare them as a whole - jq --argfile x "$1" --argfile y "$2" \ - -n 'def replace: map(if (.state == "Changed") then (.state="Added") else . end); - def normalize: .storage = (.storage | replace | sort_by(.key)); - ($x | map(normalize)) as $x - | ($y | map(normalize)) as $y - | $x | range(length) | . as $i | select($x[$i] != $y[$i]) | $x[$i].block | halt_error(1)' -} - -if [ -f "$ARG1" ] && [ -f "$ARG2" ]; then - compare "$ARG1" "$ARG2" - if [ $? -ne 0 ]; then - echo failed - exit 1 - fi - - exit 0 -fi - -if [ ! -d "$ARG1" ] || [ ! -d "$ARG2" ]; then - echo both arguments must have the same type and exist - exit 1 -fi - -FIRST=$3 -if [ -z "$FIRST" ]; then - FIRST=1 -fi - -LAST=$4 -if [ -z "$LAST" ]; then - LAST=40 # 40_00000 block -fi - -# directories contain 100k blocks -for i in `seq $FIRST $LAST`; do - dir=BlockStorage_${i}00000 - echo Processing directory $dir - - # files are grouped by 1k blocks - for j in `seq $(((i-1)*100 + 1)) $((i*100))`; do - file=dump-block-${j}000.json - compare "$ARG1/$dir/$file" "$ARG2/$dir/$file" - if [ $? -ne 0 ]; then - echo failed on file $dir/$file - exit 1 - fi - done -done diff --git a/scripts/compare-dumps.go b/scripts/compare-dumps.go new file mode 100644 index 000000000..268806e6d --- /dev/null +++ b/scripts/compare-dumps.go @@ -0,0 +1,167 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + + "github.com/urfave/cli" +) + +type dump []blockDump + +type blockDump struct { + Block uint32 `json:"block"` + Size int `json:"size"` + Storage []storageOp `json:"storage"` +} + +type storageOp struct { + State string `json:"state"` + Key string `json:"key"` + Value string `json:"value,omitempty"` +} + +func readFile(path string) (dump, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + d := make(dump, 0) + if err := json.Unmarshal(data, &d); err != nil { + return nil, err + } + return d, err +} + +func (d dump) normalize() { + for i := range d { + for j := range d[i].Storage { + if d[i].Storage[j].State == "Changed" { + d[i].Storage[j].State = "Added" + } + } + sort.Slice(d[i].Storage, func(k, l int) bool { + return d[i].Storage[k].Key < d[i].Storage[l].Key + }) + } + // assume that d is already sorted by Block +} + +func compare(a, b string) error { + dumpA, err := readFile(a) + if err != nil { + return fmt.Errorf("reading file %s: %v", a, err) + } + dumpB, err := readFile(b) + if err != nil { + return fmt.Errorf("reading file %s: %v", b, err) + } + dumpA.normalize() + dumpB.normalize() + if len(dumpA) != len(dumpB) { + return fmt.Errorf("dump files differ in size: %d vs %d", len(dumpA), len(dumpB)) + } + for i := range dumpA { + blockA := &dumpA[i] + blockB := &dumpB[i] + if blockA.Block != blockB.Block { + return fmt.Errorf("block number mismatch: %d vs %d", blockA.Block, blockB.Block) + } + if blockA.Size != blockB.Size { + return fmt.Errorf("block %d, changes number mismatch: %d vs %d", blockA.Block, blockA.Size, blockB.Size) + } + if len(blockA.Storage) != len(blockB.Storage) { + return fmt.Errorf("block %d, changes length mismatch: %d vs %d", blockA.Block, len(blockA.Storage), len(blockB.Storage)) + } + fail := false + for j := range blockA.Storage { + if blockA.Storage[j].Key != blockB.Storage[j].Key { + return fmt.Errorf("block %d: key mismatch: %s vs %s", blockA.Block, blockA.Storage[j].Key, blockB.Storage[j].Key) + } + if blockA.Storage[j].State != blockB.Storage[j].State { + return fmt.Errorf("block %d: state mismatch for key %s: %s vs %s", blockA.Block, blockA.Storage[j].Key, blockA.Storage[j].State, blockB.Storage[j].State) + } + if blockA.Storage[j].Value != blockB.Storage[j].Value { + fail = true + fmt.Printf("block %d: value mismatch for key %s: %s vs %s\n", blockA.Block, blockA.Storage[j].Key, blockA.Storage[j].Value, blockB.Storage[j].Value) + } + } + if fail { + return errors.New("fail") + } + } + return nil +} + +func cliMain(c *cli.Context) error { + a := c.Args().Get(0) + b := c.Args().Get(1) + if a == "" { + return errors.New("no arguments given") + } + if b == "" { + return errors.New("missing second argument") + } + fa, err := os.Open(a) + if err != nil { + return err + } + defer fa.Close() + fb, err := os.Open(b) + if err != nil { + return err + } + defer fb.Close() + + astat, err := fa.Stat() + if err != nil { + return err + } + bstat, err := fb.Stat() + if err != nil { + return err + } + if astat.Mode().IsRegular() && bstat.Mode().IsRegular() { + return compare(a, b) + } + if astat.Mode().IsDir() && bstat.Mode().IsDir() { + for i := 0; i <= 6000000; i += 100000 { + dir := fmt.Sprintf("BlockStorage_%d", i) + fmt.Println("Processing directory", dir) + for j := i - 99000; j <= i; j += 1000 { + if j < 0 { + continue + } + fname := fmt.Sprintf("%s/dump-block-%d.json", dir, j) + + aname := filepath.Join(a, fname) + bname := filepath.Join(b, fname) + err := compare(aname, bname) + if err != nil { + return fmt.Errorf("file %s: %v", fname, err) + } + } + } + return nil + } + return errors.New("both parameters must be either dump files or directories") +} + +func main() { + ctl := cli.NewApp() + ctl.Name = "compare-dumps" + ctl.Version = "1.0" + ctl.Usage = "compare-dumps dumpDirA dumpDirB" + ctl.Action = cliMain + + if err := ctl.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + fmt.Fprintln(os.Stderr, ctl.Usage) + os.Exit(1) + } +}