neoneo-go/cli/server/dump.go

package server

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/nspcc-dev/neo-go/pkg/core/storage"
	"github.com/nspcc-dev/neo-go/pkg/util"
)

type dump []blockDump

type blockDump struct {
	Block   uint32      `json:"block"`
	Size    int         `json:"size"`
	Storage []storageOp `json:"storage"`
}

type storageOp struct {
	State string `json:"state"`
	Key   string `json:"key"`
	Value string `json:"value,omitempty"`
}

// NEO has some differences of key storing.
// out format: script hash in LE + key
// neo format: script hash in BE + byte(0) + key with 0 between every 16 bytes, padded to len 16.
func toNeoStorageKey(key []byte) []byte {
	if len(key) < util.Uint160Size {
		panic("invalid key in storage")
	}

	var nkey []byte
	for i := util.Uint160Size - 1; i >= 0; i-- {
		nkey = append(nkey, key[i])
	}

	key = key[util.Uint160Size:]

	index := 0
	remain := len(key)
	for remain >= 16 {
		nkey = append(nkey, key[index:index+16]...)
		nkey = append(nkey, 0)
		index += 16
		remain -= 16
	}

	if remain > 0 {
		nkey = append(nkey, key[index:]...)
	}

	padding := 16 - remain
	for i := 0; i < padding; i++ {
		nkey = append(nkey, 0)
	}

	nkey = append(nkey, byte(padding))

	return nkey
}

// batchToMap converts batch to a map so that JSON is compatible
// with https://github.com/NeoResearch/neo-storage-audit/
func batchToMap(index uint32, batch *storage.MemBatch) blockDump {
	size := len(batch.Put) + len(batch.Deleted)
	ops := make([]storageOp, 0, size)
	for i := range batch.Put {
		key := batch.Put[i].Key
		if len(key) == 0 || key[0] != byte(storage.STStorage) {
			continue
		}

		op := "Added"
		if batch.Put[i].Exists {
			op = "Changed"
		}

		key = toNeoStorageKey(key[1:])
		ops = append(ops, storageOp{
			State: op,
			Key:   hex.EncodeToString(key),
			Value: "00" + hex.EncodeToString(batch.Put[i].Value),
		})
	}

	for i := range batch.Deleted {
		key := batch.Deleted[i].Key
		if len(key) == 0 || key[0] != byte(storage.STStorage) || !batch.Deleted[i].Exists {
			continue
		}

		key = toNeoStorageKey(key[1:])
		ops = append(ops, storageOp{
			State: "Deleted",
			Key:   hex.EncodeToString(key),
		})
	}

	return blockDump{
		Block:   index,
		Size:    len(ops),
		Storage: ops,
	}
}

func newDump() *dump {
	return new(dump)
}

func (d *dump) add(index uint32, batch *storage.MemBatch) {
	m := batchToMap(index, batch)
	*d = append(*d, m)
}

func (d *dump) tryPersist(prefix string, index uint32) error {
	if len(*d) == 0 {
		return nil
	}
	path, err := getPath(prefix, index)
	if err != nil {
		return err
	}
	old, err := readFile(path)
	if err == nil {
		*old = append(*old, *d...)
	} else {
		old = d
	}
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	enc := json.NewEncoder(f)
	enc.SetIndent("", " ")
	if err := enc.Encode(*old); err != nil {
		return err
	}

	*d = (*d)[:0]

	return nil
}

func readFile(path string) (*dump, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	d := newDump()
	if err := json.Unmarshal(data, d); err != nil {
		return nil, err
	}
	return d, err
}

// getPath returns filename for storing blocks up to index.
// Directory structure is the following:
// https://github.com/NeoResearch/neo-storage-audit#folder-organization-where-to-find-the-desired-block
// Dir `BlockStorage_$DIRNO` contains blocks up to $DIRNO (from $DIRNO-100k)
// Inside it there are files grouped by 1k blocks.
// File dump-block-$FILENO.json contains blocks from $FILENO-999, $FILENO
// Example: file `BlockStorage_100000/dump-block-6000.json` contains blocks from 5001 to 6000.
func getPath(prefix string, index uint32) (string, error) {
	dirN := (index-1)/100000 + 1
	dir := fmt.Sprintf("BlockStorage_%d00000", dirN)

	path := filepath.Join(prefix, dir)
	info, err := os.Stat(path)
	if os.IsNotExist(err) {
		err := os.MkdirAll(path, os.ModePerm)
		if err != nil {
			return "", err
		}
	} else if !info.IsDir() {
		return "", fmt.Errorf("file `%s` is not a directory", path)
	}

	fileN := (index-1)/1000 + 1
	file := fmt.Sprintf("dump-block-%d000.json", fileN)
	return filepath.Join(path, file), nil
}