cli: append blocks to existing dumps

cli/server/dump.go

@@ -4,6 +4,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -111,16 +112,28 @@ func (d *dump) add(index uint32, batch *storage.MemBatch) {
 }
 
 func (d *dump) tryPersist(prefix string, index uint32) error {
-	f, err := createFile(prefix, index)
+	if len(*d) == 0 {
+		return nil
+	}
+	path, err := getPath(prefix, index)
+	if err != nil {
+		return err
+	}
+	old, err := readFile(path)
+	if err == nil {
+		*old = append(*old, *d...)
+	} else {
+		old = d
+	}
+	f, err := os.Create(path)
 	if err != nil {
 		return err
 	}
-
 	defer f.Close()
 
 	enc := json.NewEncoder(f)
 	enc.SetIndent("", " ")
-	if err := enc.Encode(*d); err != nil {
+	if err := enc.Encode(*old); err != nil {
 		return err
 	}
 
@@ -129,14 +142,26 @@ func (d *dump) tryPersist(prefix string, index uint32) error {
 	return nil
 }
 
-// createFile creates directory and file in it for storing blocks up to index.
+func readFile(path string) (*dump, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	d := newDump()
+	if err := json.Unmarshal(data, d); err != nil {
+		return nil, err
+	}
+	return d, err
+}
+
+// getPath returns filename for storing blocks up to index.
 // Directory structure is the following:
 // https://github.com/NeoResearch/neo-storage-audit#folder-organization-where-to-find-the-desired-block
 // Dir `BlockStorage_$DIRNO` contains blocks up to $DIRNO (from $DIRNO-100k)
 // Inside it there are files grouped by 1k blocks.
 // File dump-block-$FILENO.json contains blocks from $FILENO-999, $FILENO
 // Example: file `BlockStorage_100000/dump-block-6000.json` contains blocks from 5001 to 6000.
-func createFile(prefix string, index uint32) (*os.File, error) {
+func getPath(prefix string, index uint32) (string, error) {
 	dirN := (index-1)/100000 + 1
 	dir := fmt.Sprintf("BlockStorage_%d00000", dirN)
 
@@ -145,15 +170,13 @@ func createFile(prefix string, index uint32) (*os.File, error) {
 	if os.IsNotExist(err) {
 		err := os.MkdirAll(path, os.ModePerm)
 		if err != nil {
-			return nil, err
+			return "", err
 		}
 	} else if !info.IsDir() {
-		return nil, fmt.Errorf("file `%s` is not a directory", path)
+		return "", fmt.Errorf("file `%s` is not a directory", path)
 	}
 
 	fileN := (index-1)/1000 + 1
 	file := fmt.Sprintf("dump-block-%d000.json", fileN)
-	path = filepath.Join(path, file)
+	return filepath.Join(path, file), nil
-
-	return os.Create(path)
 }