From 37571162a0f5e85b859bf4cd13b7187b9e262113 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Mon, 3 Oct 2022 15:05:34 +0300 Subject: [PATCH 01/27] cli: move config path flag to options package --- cli/options/options.go | 17 +++++++++++++++++ cli/server/server.go | 22 +++++----------------- cli/server/server_test.go | 5 +++-- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/cli/options/options.go b/cli/options/options.go index cb23cb72c..8d6b5a4eb 100644 --- a/cli/options/options.go +++ b/cli/options/options.go @@ -9,6 +9,7 @@ import ( "strconv" "time" + "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/config/netmode" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/rpcclient" @@ -52,6 +53,12 @@ var Historic = cli.StringFlag{ Usage: "Use historic state (height, block hash or state root hash)", } +// Config is a flag for commands that use node configuration. +var Config = cli.StringFlag{ + Name: "config-path", + Usage: "path to directory with configuration files", +} + var errNoEndpoint = errors.New("no RPC endpoint specified, use option '--" + RPCEndpointFlag + "' or '-r'") var errInvalidHistoric = errors.New("invalid 'historic' parameter, neither a block number, nor a block/state hash") @@ -128,3 +135,13 @@ func GetRPCWithInvoker(gctx context.Context, ctx *cli.Context, signers []transac } return c, inv, err } + +// GetConfigFromContext looks at the path and the mode flags in the given config and +// returns an appropriate config. +func GetConfigFromContext(ctx *cli.Context) (config.Config, error) { + configPath := "./config" + if argCp := ctx.String("config-path"); argCp != "" { + configPath = argCp + } + return config.Load(configPath, GetNetwork(ctx)) +} diff --git a/cli/server/server.go b/cli/server/server.go index 827b853b7..8ae5b8136 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -43,9 +43,7 @@ var ( // NewCommands returns 'node' command. func NewCommands() []cli.Command { - var cfgFlags = []cli.Flag{ - cli.StringFlag{Name: "config-path", Usage: "path to directory with configuration files"}, - } + cfgFlags := []cli.Flag{options.Config} cfgFlags = append(cfgFlags, options.Network...) var cfgWithCountFlags = make([]cli.Flag, len(cfgFlags)) copy(cfgWithCountFlags, cfgFlags) @@ -128,16 +126,6 @@ func newGraceContext() context.Context { return ctx } -// getConfigFromContext looks at the path and the mode flags in the given config and -// returns an appropriate config. -func getConfigFromContext(ctx *cli.Context) (config.Config, error) { - configPath := "./config" - if argCp := ctx.String("config-path"); argCp != "" { - configPath = argCp - } - return config.Load(configPath, options.GetNetwork(ctx)) -} - // handleLoggingParams reads logging parameters. // If a user selected debug level -- function enables it. // If logPath is configured -- function creates a dir and a file for logging. @@ -233,7 +221,7 @@ func dumpDB(ctx *cli.Context) error { if err := cmdargs.EnsureNone(ctx); err != nil { return err } - cfg, err := getConfigFromContext(ctx) + cfg, err := options.GetConfigFromContext(ctx) if err != nil { return cli.NewExitError(err, 1) } @@ -286,7 +274,7 @@ func restoreDB(ctx *cli.Context) error { if err := cmdargs.EnsureNone(ctx); err != nil { return err } - cfg, err := getConfigFromContext(ctx) + cfg, err := options.GetConfigFromContext(ctx) if err != nil { return err } @@ -472,7 +460,7 @@ func startServer(ctx *cli.Context) error { return err } - cfg, err := getConfigFromContext(ctx) + cfg, err := options.GetConfigFromContext(ctx) if err != nil { return cli.NewExitError(err, 1) } @@ -549,7 +537,7 @@ Main: cancel() case sig := <-sigCh: log.Info("signal received", zap.Stringer("name", sig)) - cfgnew, err := getConfigFromContext(ctx) + cfgnew, err := options.GetConfigFromContext(ctx) if err != nil { log.Warn("can't reread the config file, signal ignored", zap.Error(err)) break // Continue working. diff --git a/cli/server/server_test.go b/cli/server/server_test.go index f3f8c107c..ff51b431b 100644 --- a/cli/server/server_test.go +++ b/cli/server/server_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "testing" + "github.com/nspcc-dev/neo-go/cli/options" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/config/netmode" "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" @@ -32,7 +33,7 @@ func TestGetConfigFromContext(t *testing.T) { set.String("config-path", "../../config", "") set.Bool("testnet", true, "") ctx := cli.NewContext(cli.NewApp(), set, nil) - cfg, err := getConfigFromContext(ctx) + cfg, err := options.GetConfigFromContext(ctx) require.NoError(t, err) require.Equal(t, netmode.TestNet, cfg.ProtocolConfiguration.Magic) } @@ -101,7 +102,7 @@ func TestInitBCWithMetrics(t *testing.T) { set.Bool("testnet", true, "") set.Bool("debug", true, "") ctx := cli.NewContext(cli.NewApp(), set, nil) - cfg, err := getConfigFromContext(ctx) + cfg, err := options.GetConfigFromContext(ctx) require.NoError(t, err) logger, closer, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) require.NoError(t, err) From 5698ce03be83d9ac56ca2b57e566bbe88692f5d5 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Mon, 3 Oct 2022 15:05:40 +0300 Subject: [PATCH 02/27] cli: move debug flag to options package --- cli/options/options.go | 6 ++++++ cli/server/server.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/options/options.go b/cli/options/options.go index 8d6b5a4eb..b2bcedc13 100644 --- a/cli/options/options.go +++ b/cli/options/options.go @@ -59,6 +59,12 @@ var Config = cli.StringFlag{ Usage: "path to directory with configuration files", } +// Debug is a flag for commands that allow node in debug mode usage. +var Debug = cli.BoolFlag{ + Name: "debug, d", + Usage: "enable debug logging (LOTS of output)", +} + var errNoEndpoint = errors.New("no RPC endpoint specified, use option '--" + RPCEndpointFlag + "' or '-r'") var errInvalidHistoric = errors.New("invalid 'historic' parameter, neither a block number, nor a block/state hash") diff --git a/cli/server/server.go b/cli/server/server.go index 8ae5b8136..88d5ff790 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -47,7 +47,7 @@ func NewCommands() []cli.Command { cfgFlags = append(cfgFlags, options.Network...) var cfgWithCountFlags = make([]cli.Flag, len(cfgFlags)) copy(cfgWithCountFlags, cfgFlags) - cfgFlags = append(cfgFlags, cli.BoolFlag{Name: "debug, d", Usage: "enable debug logging (LOTS of output)"}) + cfgFlags = append(cfgFlags, options.Debug) cfgWithCountFlags = append(cfgWithCountFlags, cli.UintFlag{ From c4c93b591e660813ce4feb1fab8b9682f670386a Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Mon, 3 Oct 2022 15:05:42 +0300 Subject: [PATCH 03/27] cli: remove debug flag from VM CLI It's not used. --- cli/vm/vm.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cli/vm/vm.go b/cli/vm/vm.go index 9a0d780ed..860a4ad02 100644 --- a/cli/vm/vm.go +++ b/cli/vm/vm.go @@ -15,9 +15,7 @@ func NewCommands() []cli.Command { Name: "vm", Usage: "start the virtual machine", Action: startVMPrompt, - Flags: []cli.Flag{ - cli.BoolFlag{Name: "debug, d"}, - }, + Flags: []cli.Flag{}, }} } From 03a1cf9f593d88f54a1407999273432ad0acbb10 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 4 Oct 2022 08:10:44 +0300 Subject: [PATCH 04/27] core: simplify newLevelDBForTesting function --- pkg/core/storage/leveldb_store_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/core/storage/leveldb_store_test.go b/pkg/core/storage/leveldb_store_test.go index 5bdb31828..178e81298 100644 --- a/pkg/core/storage/leveldb_store_test.go +++ b/pkg/core/storage/leveldb_store_test.go @@ -9,13 +9,10 @@ import ( func newLevelDBForTesting(t testing.TB) Store { ldbDir := t.TempDir() - dbConfig := dbconfig.DBConfiguration{ - Type: "leveldb", - LevelDBOptions: dbconfig.LevelDBOptions{ - DataDirectoryPath: ldbDir, - }, + opts := dbconfig.LevelDBOptions{ + DataDirectoryPath: ldbDir, } - newLevelStore, err := NewLevelDBStore(dbConfig.LevelDBOptions) + newLevelStore, err := NewLevelDBStore(opts) require.Nil(t, err, "NewLevelDBStore error") return newLevelStore } From cbdd45cc962412fdfb4154b13f5e059def3de18a Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 5 Oct 2022 08:13:12 +0300 Subject: [PATCH 05/27] core: return error on root BoltDB bucket creation if so --- pkg/core/storage/boltdb_store.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/core/storage/boltdb_store.go b/pkg/core/storage/boltdb_store.go index 7464a9fcb..59b51a49c 100644 --- a/pkg/core/storage/boltdb_store.go +++ b/pkg/core/storage/boltdb_store.go @@ -39,6 +39,9 @@ func NewBoltDBStore(cfg dbconfig.BoltDBOptions) (*BoltDBStore, error) { } return nil }) + if err != nil { + return nil, fmt.Errorf("failed to initialize BoltDB instance: %w", err) + } return &BoltDBStore{db: db}, nil } From 2f5137e9b72d2cf68ca92c6b797843b9c0dabef5 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Mon, 3 Oct 2022 15:05:44 +0300 Subject: [PATCH 06/27] core: allow RO mode for Bolt and Level --- docs/node-configuration.md | 10 ++++-- pkg/core/storage/boltdb_store.go | 38 ++++++++++++++------ pkg/core/storage/boltdb_store_test.go | 43 +++++++++++++++++++++++ pkg/core/storage/dbconfig/store_config.go | 2 ++ pkg/core/storage/leveldb_store.go | 8 +++-- pkg/core/storage/leveldb_store_test.go | 27 ++++++++++++++ 6 files changed, 113 insertions(+), 15 deletions(-) diff --git a/docs/node-configuration.md b/docs/node-configuration.md index 7536dc4d1..9f4fa8d3a 100644 --- a/docs/node-configuration.md +++ b/docs/node-configuration.md @@ -47,14 +47,20 @@ DBConfiguration: Type: leveldb LevelDBOptions: DataDirectoryPath: /chains/privnet + ReadOnly: false BoltDBOptions: FilePath: ./chains/privnet.bolt + ReadOnly: false ``` where: - `Type` is the database type (string value). Supported types: `levelDB` and `boltDB`. -- `LevelDBOptions` are settings for LevelDB. -- `BoltDBOptions` configures BoltDB. +- `LevelDBOptions` are settings for LevelDB. Includes the DB files path and ReadOnly mode toggle. + If ReadOnly mode is on, then an error will be returned on attempt to connect to unexisting or empty + database. Database doesn't allow changes in this mode, a warning will be logged on DB persist attempts. +- `BoltDBOptions` configures BoltDB. Includes the DB files path and ReadOnly mode toggle. If ReadOnly + mode is on, then an error will be returned on attempt to connect with unexisting or empty database. + Database doesn't allow changes in this mode, a warning will be logged on DB persist attempts. Only options for the specified database type will be used. diff --git a/pkg/core/storage/boltdb_store.go b/pkg/core/storage/boltdb_store.go index 59b51a49c..a874dbd8c 100644 --- a/pkg/core/storage/boltdb_store.go +++ b/pkg/core/storage/boltdb_store.go @@ -2,6 +2,7 @@ package storage import ( "bytes" + "errors" "fmt" "os" @@ -22,23 +23,38 @@ type BoltDBStore struct { // NewBoltDBStore returns a new ready to use BoltDB storage with created bucket. func NewBoltDBStore(cfg dbconfig.BoltDBOptions) (*BoltDBStore, error) { - var opts *bbolt.Options // should be exposed via BoltDBOptions if anything needed + cp := *bbolt.DefaultOptions // Do not change bbolt's global variable. + opts := &cp fileMode := os.FileMode(0600) // should be exposed via BoltDBOptions if anything needed fileName := cfg.FilePath - if err := io.MakeDirForFile(fileName, "BoltDB"); err != nil { - return nil, err + if cfg.ReadOnly { + opts.ReadOnly = true + } else { + if err := io.MakeDirForFile(fileName, "BoltDB"); err != nil { + return nil, err + } } db, err := bbolt.Open(fileName, fileMode, opts) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to open BoltDB instance: %w", err) + } + if opts.ReadOnly { + err = db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(Bucket) + if b == nil { + return errors.New("root bucket does not exist") + } + return nil + }) + } else { + err = db.Update(func(tx *bbolt.Tx) error { + _, err = tx.CreateBucketIfNotExists(Bucket) + if err != nil { + return fmt.Errorf("could not create root bucket: %w", err) + } + return nil + }) } - err = db.Update(func(tx *bbolt.Tx) error { - _, err = tx.CreateBucketIfNotExists(Bucket) - if err != nil { - return fmt.Errorf("could not create root bucket: %w", err) - } - return nil - }) if err != nil { return nil, fmt.Errorf("failed to initialize BoltDB instance: %w", err) } diff --git a/pkg/core/storage/boltdb_store_test.go b/pkg/core/storage/boltdb_store_test.go index e46c5b3d4..a877b4e38 100644 --- a/pkg/core/storage/boltdb_store_test.go +++ b/pkg/core/storage/boltdb_store_test.go @@ -1,11 +1,15 @@ package storage import ( + "os" "path/filepath" + "strings" "testing" "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" + "github.com/nspcc-dev/neo-go/pkg/io" "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" ) func newBoltStoreForTesting(t testing.TB) Store { @@ -15,3 +19,42 @@ func newBoltStoreForTesting(t testing.TB) Store { require.NoError(t, err) return boltDBStore } + +func TestROBoltDB(t *testing.T) { + d := t.TempDir() + testFileName := filepath.Join(d, "test_ro_bolt_db") + cfg := dbconfig.BoltDBOptions{ + FilePath: testFileName, + ReadOnly: true, + } + + // If DB doesn't exist, then error should be returned. + _, err := NewBoltDBStore(cfg) + require.Error(t, err) + + // Create the DB and try to open it in RO mode. + cfg.ReadOnly = false + store, err := NewBoltDBStore(cfg) + require.NoError(t, err) + require.NoError(t, store.Close()) + cfg.ReadOnly = true + + store, err = NewBoltDBStore(cfg) + require.NoError(t, err) + // Changes must be prohibited. + putErr := store.PutChangeSet(map[string][]byte{"one": []byte("one")}, nil) + require.ErrorIs(t, putErr, bbolt.ErrDatabaseReadOnly) + require.NoError(t, store.Close()) + + // Create the DB without buckets and try to open it in RO mode, an error is expected. + fileMode := os.FileMode(0600) + cfg.FilePath = filepath.Join(d, "clean_ro_bolt_db") + require.NoError(t, io.MakeDirForFile(cfg.FilePath, "BoltDB")) + db, err := bbolt.Open(cfg.FilePath, fileMode, nil) + require.NoError(t, err) + require.NoError(t, db.Close()) + + _, err = NewBoltDBStore(cfg) + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "root bucket does not exist")) +} diff --git a/pkg/core/storage/dbconfig/store_config.go b/pkg/core/storage/dbconfig/store_config.go index c2dde12b7..30945a640 100644 --- a/pkg/core/storage/dbconfig/store_config.go +++ b/pkg/core/storage/dbconfig/store_config.go @@ -13,9 +13,11 @@ type ( // LevelDBOptions configuration for LevelDB. LevelDBOptions struct { DataDirectoryPath string `yaml:"DataDirectoryPath"` + ReadOnly bool `yaml:"ReadOnly"` } // BoltDBOptions configuration for BoltDB. BoltDBOptions struct { FilePath string `yaml:"FilePath"` + ReadOnly bool `yaml:"ReadOnly"` } ) diff --git a/pkg/core/storage/leveldb_store.go b/pkg/core/storage/leveldb_store.go index f1f4930f2..be2537c6e 100644 --- a/pkg/core/storage/leveldb_store.go +++ b/pkg/core/storage/leveldb_store.go @@ -2,6 +2,7 @@ package storage import ( "errors" + "fmt" "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/syndtr/goleveldb/leveldb" @@ -21,11 +22,14 @@ type LevelDBStore struct { // initialize the database found at the given path. func NewLevelDBStore(cfg dbconfig.LevelDBOptions) (*LevelDBStore, error) { var opts = new(opt.Options) // should be exposed via LevelDBOptions if anything needed - + if cfg.ReadOnly { + opts.ReadOnly = true + opts.ErrorIfMissing = true + } opts.Filter = filter.NewBloomFilter(10) db, err := leveldb.OpenFile(cfg.DataDirectoryPath, opts) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to open LevelDB instance: %w", err) } return &LevelDBStore{ diff --git a/pkg/core/storage/leveldb_store_test.go b/pkg/core/storage/leveldb_store_test.go index 178e81298..6e9b105e7 100644 --- a/pkg/core/storage/leveldb_store_test.go +++ b/pkg/core/storage/leveldb_store_test.go @@ -5,6 +5,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/stretchr/testify/require" + "github.com/syndtr/goleveldb/leveldb" ) func newLevelDBForTesting(t testing.TB) Store { @@ -16,3 +17,29 @@ func newLevelDBForTesting(t testing.TB) Store { require.Nil(t, err, "NewLevelDBStore error") return newLevelStore } + +func TestROLevelDB(t *testing.T) { + ldbDir := t.TempDir() + opts := dbconfig.LevelDBOptions{ + DataDirectoryPath: ldbDir, + ReadOnly: true, + } + + // If DB doesn't exist, then error should be returned. + _, err := NewLevelDBStore(opts) + require.Error(t, err) + + // Create the DB and try to open it in RO mode. + opts.ReadOnly = false + store, err := NewLevelDBStore(opts) + require.NoError(t, err) + require.NoError(t, store.Close()) + opts.ReadOnly = true + + store, err = NewLevelDBStore(opts) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + // Changes must be prohibited. + putErr := store.PutChangeSet(map[string][]byte{"one": []byte("one")}, nil) + require.ErrorIs(t, putErr, leveldb.ErrReadOnly) +} From 70e59d83c9b524ebd61006544897f262928f501b Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 5 Oct 2022 09:05:36 +0300 Subject: [PATCH 07/27] docs: fix supported database types --- cli/server/cli_dump_test.go | 3 +- cli/server/cli_server_test.go | 3 +- cli/server/server_test.go | 2 +- docs/node-configuration.md | 4 +-- .../native/native_test/management_test.go | 2 +- pkg/core/storage/dbconfig/store_type.go | 11 +++++++ pkg/core/storage/store.go | 6 ++-- pkg/core/storage/store_type_test.go | 29 +++++++++++++++++++ 8 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 pkg/core/storage/dbconfig/store_type.go create mode 100644 pkg/core/storage/store_type_test.go diff --git a/cli/server/cli_dump_test.go b/cli/server/cli_dump_test.go index 4911de3bc..b743f9015 100644 --- a/cli/server/cli_dump_test.go +++ b/cli/server/cli_dump_test.go @@ -8,6 +8,7 @@ import ( "github.com/nspcc-dev/neo-go/internal/testcli" "github.com/nspcc-dev/neo-go/pkg/config" + "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -19,7 +20,7 @@ func TestDBRestoreDump(t *testing.T) { chainPath := filepath.Join(tmpDir, "neogotestchain") cfg, err := config.LoadFile(filepath.Join("..", "..", "config", "protocol.unit_testnet.yml")) require.NoError(t, err, "could not load config") - cfg.ApplicationConfiguration.DBConfiguration.Type = "leveldb" + cfg.ApplicationConfiguration.DBConfiguration.Type = dbconfig.LevelDB cfg.ApplicationConfiguration.DBConfiguration.LevelDBOptions.DataDirectoryPath = chainPath return cfg } diff --git a/cli/server/cli_server_test.go b/cli/server/cli_server_test.go index ccaf646e9..2170ec85c 100644 --- a/cli/server/cli_server_test.go +++ b/cli/server/cli_server_test.go @@ -13,6 +13,7 @@ import ( "github.com/nspcc-dev/neo-go/cli/server" "github.com/nspcc-dev/neo-go/internal/testcli" "github.com/nspcc-dev/neo-go/pkg/config" + "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -25,7 +26,7 @@ func TestServerStart(t *testing.T) { saveCfg := func(t *testing.T, f func(cfg *config.Config)) string { cfg := *ptr chainPath := filepath.Join(t.TempDir(), "neogotestchain") - cfg.ApplicationConfiguration.DBConfiguration.Type = "leveldb" + cfg.ApplicationConfiguration.DBConfiguration.Type = dbconfig.LevelDB cfg.ApplicationConfiguration.DBConfiguration.LevelDBOptions.DataDirectoryPath = chainPath f(&cfg) out, err := yaml.Marshal(cfg) diff --git a/cli/server/server_test.go b/cli/server/server_test.go index ff51b431b..36d751cb1 100644 --- a/cli/server/server_test.go +++ b/cli/server/server_test.go @@ -350,7 +350,7 @@ func TestInitBlockChain(t *testing.T) { _, err := initBlockChain(config.Config{ ApplicationConfiguration: config.ApplicationConfiguration{ DBConfiguration: dbconfig.DBConfiguration{ - Type: "inmemory", + Type: dbconfig.InMemoryDB, }, }, }, nil) diff --git a/docs/node-configuration.md b/docs/node-configuration.md index 9f4fa8d3a..fc4fb9340 100644 --- a/docs/node-configuration.md +++ b/docs/node-configuration.md @@ -53,8 +53,8 @@ DBConfiguration: ReadOnly: false ``` where: -- `Type` is the database type (string value). Supported types: `levelDB` and - `boltDB`. +- `Type` is the database type (string value). Supported types: `leveldb`, `boltdb` and + `inmemory` (not recommended for production usage). - `LevelDBOptions` are settings for LevelDB. Includes the DB files path and ReadOnly mode toggle. If ReadOnly mode is on, then an error will be returned on attempt to connect to unexisting or empty database. Database doesn't allow changes in this mode, a warning will be logged on DB persist attempts. diff --git a/pkg/core/native/native_test/management_test.go b/pkg/core/native/native_test/management_test.go index 37699f59d..68f808895 100644 --- a/pkg/core/native/native_test/management_test.go +++ b/pkg/core/native/native_test/management_test.go @@ -288,7 +288,7 @@ func TestManagement_StartFromHeight(t *testing.T) { // Create database to be able to start another chain from the same height later. ldbDir := t.TempDir() dbConfig := dbconfig.DBConfiguration{ - Type: "leveldb", + Type: dbconfig.LevelDB, LevelDBOptions: dbconfig.LevelDBOptions{ DataDirectoryPath: ldbDir, }, diff --git a/pkg/core/storage/dbconfig/store_type.go b/pkg/core/storage/dbconfig/store_type.go new file mode 100644 index 000000000..9faf08ec2 --- /dev/null +++ b/pkg/core/storage/dbconfig/store_type.go @@ -0,0 +1,11 @@ +package dbconfig + +// Available storage types. +const ( + // BoltDB represents Bolt DB storage name. + BoltDB = "boltdb" + // LevelDB represents Level DB storage name. + LevelDB = "leveldb" + // InMemoryDB represents in-memory storage name. + InMemoryDB = "inmemory" +) diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index 931d153ff..caa8a4e73 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -114,11 +114,11 @@ func NewStore(cfg dbconfig.DBConfiguration) (Store, error) { var store Store var err error switch cfg.Type { - case "leveldb": + case dbconfig.LevelDB: store, err = NewLevelDBStore(cfg.LevelDBOptions) - case "inmemory": + case dbconfig.InMemoryDB: store = NewMemoryStore() - case "boltdb": + case dbconfig.BoltDB: store, err = NewBoltDBStore(cfg.BoltDBOptions) default: return nil, fmt.Errorf("unknown storage: %s", cfg.Type) diff --git a/pkg/core/storage/store_type_test.go b/pkg/core/storage/store_type_test.go new file mode 100644 index 000000000..bb5bd8936 --- /dev/null +++ b/pkg/core/storage/store_type_test.go @@ -0,0 +1,29 @@ +package storage + +import ( + "path/filepath" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" + "github.com/stretchr/testify/require" +) + +func TestStorageNames(t *testing.T) { + tmp := t.TempDir() + cfg := dbconfig.DBConfiguration{ + LevelDBOptions: dbconfig.LevelDBOptions{ + DataDirectoryPath: filepath.Join(tmp, "level"), + }, + BoltDBOptions: dbconfig.BoltDBOptions{ + FilePath: filepath.Join(tmp, "bolt"), + }, + } + for _, name := range []string{dbconfig.BoltDB, dbconfig.LevelDB, dbconfig.InMemoryDB} { + t.Run(name, func(t *testing.T) { + cfg.Type = name + s, err := NewStore(cfg) + require.NoError(t, err) + require.NoError(t, s.Close()) + }) + } +} From a91cf2a007966b139828f38eae9b836aab8d365f Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 4 Oct 2022 09:02:03 +0300 Subject: [PATCH 08/27] core: set default SecondsPerBlock value on blockchain creation As mentioned in the node configuration docs. --- pkg/core/blockchain.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 1c14b7851..ccdfb0524 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -55,6 +55,7 @@ const ( defaultMaxBlockSystemFee = 900000000000 defaultMaxTraceableBlocks = 2102400 // 1 year of 15s blocks defaultMaxTransactionsPerBlock = 512 + defaultSecondsPerBlock = 15 // HeaderVerificationGasLimit is the maximum amount of GAS for block header verification. HeaderVerificationGasLimit = 3_00000000 // 3 GAS defaultStateSyncInterval = 40000 @@ -245,6 +246,11 @@ func NewBlockchain(s storage.Store, cfg config.ProtocolConfiguration, log *zap.L log.Info("MaxTransactionsPerBlock is not set or wrong, using default value", zap.Uint16("MaxTransactionsPerBlock", cfg.MaxTransactionsPerBlock)) } + if cfg.SecondsPerBlock == 0 { + cfg.SecondsPerBlock = defaultSecondsPerBlock + log.Info("SecondsPerBlock is not set or wrong, using default value", + zap.Int("SecondsPerBlock", cfg.SecondsPerBlock)) + } if cfg.MaxValidUntilBlockIncrement == 0 { const secondsPerDay = int(24 * time.Hour / time.Second) From 4a4600174679510b3a3c3322efc73b0516493d9c Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 4 Oct 2022 09:07:57 +0300 Subject: [PATCH 09/27] smartcontract: fix error message for CreateMultiSigRedeemScript --- pkg/smartcontract/contract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/smartcontract/contract.go b/pkg/smartcontract/contract.go index bf19a2284..90b34267f 100644 --- a/pkg/smartcontract/contract.go +++ b/pkg/smartcontract/contract.go @@ -14,7 +14,7 @@ import ( // where n is the length of publicKeys. func CreateMultiSigRedeemScript(m int, publicKeys keys.PublicKeys) ([]byte, error) { if m < 1 { - return nil, fmt.Errorf("param m cannot be smaller or equal to 1 got %d", m) + return nil, fmt.Errorf("param m cannot be smaller than 1, got %d", m) } if m > len(publicKeys) { return nil, fmt.Errorf("length of the signatures (%d) is higher then the number of public keys", m) From 0b717b0c228424809ee63e370230c32481efdee4 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 5 Oct 2022 12:30:54 +0300 Subject: [PATCH 10/27] vm: move vm CLI to cli/vm package --- cli/util/convert.go | 2 +- {pkg/vm/cli => cli/vm}/cli.go | 2 +- {pkg/vm/cli => cli/vm}/cli_test.go | 4 ++-- cli/vm/vm.go | 3 +-- 4 files changed, 5 insertions(+), 6 deletions(-) rename {pkg/vm/cli => cli/vm}/cli.go (99%) rename {pkg/vm/cli => cli/vm}/cli_test.go (99%) diff --git a/cli/util/convert.go b/cli/util/convert.go index 2f75be9bd..1fb910216 100644 --- a/cli/util/convert.go +++ b/cli/util/convert.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/nspcc-dev/neo-go/cli/options" - vmcli "github.com/nspcc-dev/neo-go/pkg/vm/cli" + vmcli "github.com/nspcc-dev/neo-go/cli/vm" "github.com/urfave/cli" ) diff --git a/pkg/vm/cli/cli.go b/cli/vm/cli.go similarity index 99% rename from pkg/vm/cli/cli.go rename to cli/vm/cli.go index ae848670a..f7f210966 100644 --- a/pkg/vm/cli/cli.go +++ b/cli/vm/cli.go @@ -1,4 +1,4 @@ -package cli +package vm import ( "bytes" diff --git a/pkg/vm/cli/cli_test.go b/cli/vm/cli_test.go similarity index 99% rename from pkg/vm/cli/cli_test.go rename to cli/vm/cli_test.go index 98f6afacb..b49e4e274 100644 --- a/pkg/vm/cli/cli_test.go +++ b/cli/vm/cli_test.go @@ -1,4 +1,4 @@ -package cli +package vm import ( "bytes" @@ -251,7 +251,7 @@ go 1.17`) require ( github.com/nspcc-dev/neo-go/pkg/interop v0.0.0 ) -replace github.com/nspcc-dev/neo-go/pkg/interop => ` + filepath.Join(wd, "../../interop") + ` +replace github.com/nspcc-dev/neo-go/pkg/interop => ` + filepath.Join(wd, "../../pkg/interop") + ` go 1.17`) require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "go.mod"), goMod, os.ModePerm)) diff --git a/cli/vm/vm.go b/cli/vm/vm.go index 860a4ad02..a5faf4441 100644 --- a/cli/vm/vm.go +++ b/cli/vm/vm.go @@ -5,7 +5,6 @@ import ( "github.com/chzyer/readline" "github.com/nspcc-dev/neo-go/cli/cmdargs" - vmcli "github.com/nspcc-dev/neo-go/pkg/vm/cli" "github.com/urfave/cli" ) @@ -23,6 +22,6 @@ func startVMPrompt(ctx *cli.Context) error { if err := cmdargs.EnsureNone(ctx); err != nil { return err } - p := vmcli.NewWithConfig(true, os.Exit, &readline.Config{}) + p := NewWithConfig(true, os.Exit, &readline.Config{}) return p.Run() } From 513821cfff00d2961a760c5a91988e084cd9cbec Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Mon, 3 Oct 2022 15:05:48 +0300 Subject: [PATCH 11/27] vm: allow to provide state for VM CLI Close #2528. Also, add new simple testchain as an analogue for basicchain. --- cli/vm/cli.go | 91 +++++++++++++++++++++++++++--------- cli/vm/cli_test.go | 70 ++++++++++++++++++++++++++- cli/vm/vm.go | 25 +++++++++- internal/basicchain/basic.go | 25 ++++++++++ 4 files changed, 186 insertions(+), 25 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index f7f210966..d463ffcdb 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -19,20 +19,27 @@ import ( "github.com/kballard/go-shellquote" "github.com/nspcc-dev/neo-go/pkg/compiler" "github.com/nspcc-dev/neo-go/pkg/config" + "github.com/nspcc-dev/neo-go/pkg/core" + "github.com/nspcc-dev/neo-go/pkg/core/interop" + "github.com/nspcc-dev/neo-go/pkg/core/storage" + "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/encoding/bigint" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/nspcc-dev/neo-go/pkg/vm" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/urfave/cli" + "go.uber.org/zap" ) const ( - vmKey = "vm" + chainKey = "chain" + icKey = "ic" manifestKey = "manifest" exitFuncKey = "exitFunc" readlineInstanceKey = "readlineKey" @@ -245,26 +252,20 @@ var ( // VMCLI object for interacting with the VM. type VMCLI struct { - vm *vm.VM + chain *core.Blockchain shell *cli.App } -// New returns a new VMCLI object. -func New() *VMCLI { - return NewWithConfig(true, os.Exit, &readline.Config{ - Prompt: "\033[32mNEO-GO-VM >\033[0m ", // green prompt ^^ - }) -} - -// NewWithConfig returns new VMCLI instance using provided config. -func NewWithConfig(printLogotype bool, onExit func(int), c *readline.Config) *VMCLI { +// NewWithConfig returns new VMCLI instance using provided config and (optionally) +// provided node config for state-backed VM. +func NewWithConfig(printLogotype bool, onExit func(int), c *readline.Config, cfg config.Config) (*VMCLI, error) { if c.AutoComplete == nil { // Autocomplete commands/flags on TAB. c.AutoComplete = completer } l, err := readline.NewEx(c) if err != nil { - panic(err) + return nil, fmt.Errorf("failed to create readline instance: %w", err) } ctl := cli.NewApp() ctl.Name = "VM CLI" @@ -284,20 +285,41 @@ func NewWithConfig(printLogotype bool, onExit func(int), c *readline.Config) *VM ctl.Commands = commands + store, err := storage.NewStore(cfg.ApplicationConfiguration.DBConfiguration) + if err != nil { + writeErr(ctl.ErrWriter, fmt.Errorf("failed to open DB, clean in-memory storage will be used: %w", err)) + cfg.ApplicationConfiguration.DBConfiguration.Type = dbconfig.InMemoryDB + store = storage.NewMemoryStore() + } + + exitF := func(i int) { + _ = store.Close() + onExit(i) + } + + log := zap.NewNop() + chain, err := core.NewBlockchain(store, cfg.ProtocolConfiguration, log) + if err != nil { + return nil, cli.NewExitError(fmt.Errorf("could not initialize blockchain: %w", err), 1) + } + // Do not run chain, we need only state-related functionality from it. + ic := chain.GetTestVM(trigger.Application, nil, nil) + vmcli := VMCLI{ - vm: vm.New(), + chain: chain, shell: ctl, } vmcli.shell.Metadata = map[string]interface{}{ - vmKey: vmcli.vm, + chainKey: chain, + icKey: ic, manifestKey: new(manifest.Manifest), - exitFuncKey: onExit, + exitFuncKey: exitF, readlineInstanceKey: l, printLogoKey: printLogotype, } changePrompt(vmcli.shell) - return &vmcli + return &vmcli, nil } func getExitFuncFromContext(app *cli.App) func(int) { @@ -309,12 +331,15 @@ func getReadlineInstanceFromContext(app *cli.App) *readline.Instance { } func getVMFromContext(app *cli.App) *vm.VM { - return app.Metadata[vmKey].(*vm.VM) + return getInteropContextFromContext(app).VM } -func setVMInContext(app *cli.App, v *vm.VM) { - old := getVMFromContext(app) - *old = *v +func getChainFromContext(app *cli.App) *core.Blockchain { + return app.Metadata[chainKey].(*core.Blockchain) +} + +func getInteropContextFromContext(app *cli.App) *interop.Context { + return app.Metadata[icKey].(*interop.Context) } func getManifestFromContext(app *cli.App) *manifest.Manifest { @@ -325,6 +350,10 @@ func getPrintLogoFromContext(app *cli.App) bool { return app.Metadata[printLogoKey].(bool) } +func setInteropContextInContext(app *cli.App, ic *interop.Context) { + app.Metadata[icKey] = ic +} + func setManifestInContext(app *cli.App, m *manifest.Manifest) { old := getManifestFromContext(app) *old = *m @@ -340,6 +369,7 @@ func checkVMIsReady(app *cli.App) bool { } func handleExit(c *cli.Context) error { + finalizeInteropContext(c.App) l := getReadlineInstanceFromContext(c.App) _ = l.Close() exit := getExitFuncFromContext(c.App) @@ -419,6 +449,7 @@ func handleSlots(c *cli.Context) error { } func handleLoadNEF(c *cli.Context) error { + resetInteropContext(c.App) v := getVMFromContext(c.App) args := c.Args() if len(args) < 2 { @@ -438,6 +469,7 @@ func handleLoadNEF(c *cli.Context) error { } func handleLoadBase64(c *cli.Context) error { + resetInteropContext(c.App) v := getVMFromContext(c.App) args := c.Args() if len(args) < 1 { @@ -454,6 +486,7 @@ func handleLoadBase64(c *cli.Context) error { } func handleLoadHex(c *cli.Context) error { + resetInteropContext(c.App) v := getVMFromContext(c.App) args := c.Args() if len(args) < 1 { @@ -470,6 +503,7 @@ func handleLoadHex(c *cli.Context) error { } func handleLoadGo(c *cli.Context) error { + resetInteropContext(c.App) v := getVMFromContext(c.App) args := c.Args() if len(args) < 1 { @@ -496,11 +530,26 @@ func handleLoadGo(c *cli.Context) error { } func handleReset(c *cli.Context) error { - setVMInContext(c.App, vm.New()) + resetInteropContext(c.App) changePrompt(c.App) return nil } +// finalizeInteropContext calls finalizer for the current interop context. +func finalizeInteropContext(app *cli.App) { + ic := getInteropContextFromContext(app) + ic.Finalize() +} + +// resetInteropContext calls finalizer for current interop context and replaces +// it with the newly created one. +func resetInteropContext(app *cli.App) { + finalizeInteropContext(app) + bc := getChainFromContext(app) + newIc := bc.GetTestVM(trigger.Application, nil, nil) + setInteropContextInContext(app, newIc) +} + func getManifestFromFile(name string) (*manifest.Manifest, error) { bs, err := os.ReadFile(name) if err != nil { diff --git a/cli/vm/cli_test.go b/cli/vm/cli_test.go index b49e4e274..a84141930 100644 --- a/cli/vm/cli_test.go +++ b/cli/vm/cli_test.go @@ -15,12 +15,18 @@ import ( "time" "github.com/chzyer/readline" + "github.com/nspcc-dev/neo-go/internal/basicchain" "github.com/nspcc-dev/neo-go/internal/random" "github.com/nspcc-dev/neo-go/pkg/compiler" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/core/interop/interopnames" + "github.com/nspcc-dev/neo-go/pkg/core/storage" + "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/neotest" + "github.com/nspcc-dev/neo-go/pkg/neotest/chain" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm" "github.com/nspcc-dev/neo-go/pkg/vm/emit" @@ -64,12 +70,27 @@ func newTestVMCLI(t *testing.T) *executor { } func newTestVMCLIWithLogo(t *testing.T, printLogo bool) *executor { + return newTestVMCLIWithLogoAndCustomConfig(t, printLogo, nil) +} + +func newTestVMCLIWithLogoAndCustomConfig(t *testing.T, printLogo bool, cfg *config.Config) *executor { e := &executor{ in: &readCloser{Buffer: *bytes.NewBuffer(nil)}, out: bytes.NewBuffer(nil), ch: make(chan struct{}), } - e.cli = NewWithConfig(printLogo, + var c config.Config + if cfg == nil { + configPath := "../../config/protocol.unit_testnet.single.yml" + var err error + c, err = config.LoadFile(configPath) + require.NoError(t, err, "could not load chain config") + c.ApplicationConfiguration.DBConfiguration.Type = dbconfig.InMemoryDB + } else { + c = *cfg + } + var err error + e.cli, err = NewWithConfig(printLogo, func(int) { e.exit.Store(true) }, &readline.Config{ Prompt: "", @@ -79,10 +100,40 @@ func newTestVMCLIWithLogo(t *testing.T, printLogo bool) *executor { FuncIsTerminal: func() bool { return false }, - }) + }, c) + require.NoError(t, err) return e } +func newTestVMClIWithState(t *testing.T) *executor { + // Firstly create a DB with chain, save and close it. + path := t.TempDir() + opts := dbconfig.LevelDBOptions{ + DataDirectoryPath: path, + } + store, err := storage.NewLevelDBStore(opts) + require.NoError(t, err) + customConfig := func(c *config.ProtocolConfiguration) { + c.StateRootInHeader = true // Need for P2PStateExchangeExtensions check. + c.P2PSigExtensions = true // Need for basic chain initializer. + } + bc, validators, committee, err := chain.NewMultiWithCustomConfigAndStoreNoCheck(t, customConfig, store) + require.NoError(t, err) + go bc.Run() + e := neotest.NewExecutor(t, bc, validators, committee) + basicchain.InitSimple(t, "../../", e) + bc.Close() + + // After that create VMCLI backed by created chain. + configPath := "../../config/protocol.unit_testnet.yml" + cfg, err := config.LoadFile(configPath) + require.NoError(t, err) + cfg.ApplicationConfiguration.DBConfiguration.Type = dbconfig.LevelDB + cfg.ApplicationConfiguration.DBConfiguration.LevelDBOptions = opts + cfg.ProtocolConfiguration.StateRootInHeader = true + return newTestVMCLIWithLogoAndCustomConfig(t, false, &cfg) +} + func (e *executor) runProg(t *testing.T, commands ...string) { e.runProgWithTimeout(t, 4*time.Second, commands...) } @@ -662,3 +713,18 @@ func TestReset(t *testing.T) { e.checkNextLine(t, "") e.checkError(t, fmt.Errorf("VM is not ready: no program loaded")) } + +func TestRunWithState(t *testing.T) { + e := newTestVMClIWithState(t) + + // Ensure that state is properly loaded and on-chain contract can be called. + script := io.NewBufBinWriter() + h, err := e.cli.chain.GetContractScriptHash(1) // examples/storage/storage.go + require.NoError(t, err) + emit.AppCall(script.BinWriter, h, "put", callflag.All, 3, 3) + e.runProg(t, + "loadhex "+hex.EncodeToString(script.Bytes()), + "run") + e.checkNextLine(t, "READY: loaded 37 instructions") + e.checkStack(t, 3) +} diff --git a/cli/vm/vm.go b/cli/vm/vm.go index a5faf4441..23b9938de 100644 --- a/cli/vm/vm.go +++ b/cli/vm/vm.go @@ -1,20 +1,25 @@ package vm import ( + "fmt" "os" "github.com/chzyer/readline" "github.com/nspcc-dev/neo-go/cli/cmdargs" + "github.com/nspcc-dev/neo-go/cli/options" + "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/urfave/cli" ) // NewCommands returns 'vm' command. func NewCommands() []cli.Command { + cfgFlags := []cli.Flag{options.Config} + cfgFlags = append(cfgFlags, options.Network...) return []cli.Command{{ Name: "vm", Usage: "start the virtual machine", Action: startVMPrompt, - Flags: []cli.Flag{}, + Flags: cfgFlags, }} } @@ -22,6 +27,22 @@ func startVMPrompt(ctx *cli.Context) error { if err := cmdargs.EnsureNone(ctx); err != nil { return err } - p := NewWithConfig(true, os.Exit, &readline.Config{}) + + cfg, err := options.GetConfigFromContext(ctx) + if err != nil { + return cli.NewExitError(err, 1) + } + if ctx.NumFlags() == 0 { + cfg.ApplicationConfiguration.DBConfiguration.Type = dbconfig.InMemoryDB + } + if cfg.ApplicationConfiguration.DBConfiguration.Type != dbconfig.InMemoryDB { + cfg.ApplicationConfiguration.DBConfiguration.LevelDBOptions.ReadOnly = true + cfg.ApplicationConfiguration.DBConfiguration.BoltDBOptions.ReadOnly = true + } + + p, err := NewWithConfig(true, os.Exit, &readline.Config{}, cfg) + if err != nil { + return cli.NewExitError(fmt.Errorf("failed to create VM CLI: %w", err), 1) + } return p.Run() } diff --git a/internal/basicchain/basic.go b/internal/basicchain/basic.go index 304ff53c8..944008bfa 100644 --- a/internal/basicchain/basic.go +++ b/internal/basicchain/basic.go @@ -23,6 +23,31 @@ import ( const neoAmount = 99999000 +// InitSimple initializes chain with single contract storing several storage values. +// It's not as complicated as chain got after Init and may be used for tests where +// chain with a small amount of data is needed and for historical functionality testing. +// Needs a path to the root directory. +func InitSimple(t *testing.T, rootpath string, e *neotest.Executor) { + // examplesPrefix is a prefix of the example smart-contracts. + var examplesPrefix = filepath.Join(rootpath, "examples") + + // Block #1: deploy storage contract (examples/storage/storage.go). + _, storageHash := newDeployTx(t, e, e.Validator, + filepath.Join(examplesPrefix, "storage", "storage.go"), + filepath.Join(examplesPrefix, "storage", "storage.yml"), + true) + + // Block #2: put (1, 1) kv pair. + storageValidatorInvoker := e.ValidatorInvoker(storageHash) + storageValidatorInvoker.Invoke(t, 1, "put", 1, 1) + + // Block #3: put (2, 2) kv pair. + storageValidatorInvoker.Invoke(t, 2, "put", 2, 2) + + // Block #4: update (1, 1) -> (1, 2). + storageValidatorInvoker.Invoke(t, 1, "put", 1, 2) +} + // Init pushes some predefined set of transactions into the given chain, it needs a path to // the root project directory. func Init(t *testing.T, rootpath string, e *neotest.Executor) { From 33ae8d0ddc9d2aa42a1663589df46afeb45bab42 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 4 Oct 2022 13:19:18 +0300 Subject: [PATCH 12/27] vm: clear manifest on VM CLI reset Fix the bug when outdated manifest was stored after new program was loaded. --- cli/vm/cli.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index d463ffcdb..b57c6c81c 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -355,8 +355,7 @@ func setInteropContextInContext(app *cli.App, ic *interop.Context) { } func setManifestInContext(app *cli.App, m *manifest.Manifest) { - old := getManifestFromContext(app) - *old = *m + app.Metadata[manifestKey] = m } func checkVMIsReady(app *cli.App) bool { @@ -449,7 +448,7 @@ func handleSlots(c *cli.Context) error { } func handleLoadNEF(c *cli.Context) error { - resetInteropContext(c.App) + resetState(c.App) v := getVMFromContext(c.App) args := c.Args() if len(args) < 2 { @@ -469,7 +468,7 @@ func handleLoadNEF(c *cli.Context) error { } func handleLoadBase64(c *cli.Context) error { - resetInteropContext(c.App) + resetState(c.App) v := getVMFromContext(c.App) args := c.Args() if len(args) < 1 { @@ -486,7 +485,7 @@ func handleLoadBase64(c *cli.Context) error { } func handleLoadHex(c *cli.Context) error { - resetInteropContext(c.App) + resetState(c.App) v := getVMFromContext(c.App) args := c.Args() if len(args) < 1 { @@ -503,7 +502,7 @@ func handleLoadHex(c *cli.Context) error { } func handleLoadGo(c *cli.Context) error { - resetInteropContext(c.App) + resetState(c.App) v := getVMFromContext(c.App) args := c.Args() if len(args) < 1 { @@ -530,7 +529,7 @@ func handleLoadGo(c *cli.Context) error { } func handleReset(c *cli.Context) error { - resetInteropContext(c.App) + resetState(c.App) changePrompt(c.App) return nil } @@ -550,6 +549,18 @@ func resetInteropContext(app *cli.App) { setInteropContextInContext(app, newIc) } +// resetManifest removes manifest from app context. +func resetManifest(app *cli.App) { + setManifestInContext(app, nil) +} + +// resetState resets state of the app (clear interop context and manifest) so that it's ready +// to load new program. +func resetState(app *cli.App) { + resetInteropContext(app) + resetManifest(app) +} + func getManifestFromFile(name string) (*manifest.Manifest, error) { bs, err := os.ReadFile(name) if err != nil { @@ -580,6 +591,9 @@ func handleRun(c *cli.Context) error { return err } if runCurrent { + if m == nil { + return fmt.Errorf("manifest is not loaded; either use 'run' command to run loaded script from the start or use 'loadgo' and 'loadnef' commands to provide manifest") + } md := m.ABI.GetMethod(args[0], len(params)) if md == nil { return fmt.Errorf("%w: method not found", ErrInvalidParameter) From f45d8fc08d261e208a3e3768f8f7fec3a6a275d6 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 4 Oct 2022 09:46:04 +0300 Subject: [PATCH 13/27] vm: remove default syscall handler It's not needed anymore. Close #1075. --- pkg/vm/interop.go | 66 ----------------------------------------------- pkg/vm/vm.go | 5 ++-- pkg/vm/vm_test.go | 14 ++++++++++ 3 files changed, 17 insertions(+), 68 deletions(-) delete mode 100644 pkg/vm/interop.go diff --git a/pkg/vm/interop.go b/pkg/vm/interop.go deleted file mode 100644 index 27156b31f..000000000 --- a/pkg/vm/interop.go +++ /dev/null @@ -1,66 +0,0 @@ -package vm - -import ( - "errors" - "fmt" - "sort" - - "github.com/nspcc-dev/neo-go/pkg/core/interop/interopnames" - "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" -) - -// interopIDFuncPrice adds an ID to the InteropFuncPrice. -type interopIDFuncPrice struct { - ID uint32 - Func func(vm *VM) error - Price int64 - RequiredFlags callflag.CallFlag -} - -var defaultVMInterops = []interopIDFuncPrice{ - {ID: interopnames.ToID([]byte(interopnames.SystemRuntimeLog)), - Func: runtimeLog, Price: 1 << 15, RequiredFlags: callflag.AllowNotify}, - {ID: interopnames.ToID([]byte(interopnames.SystemRuntimeNotify)), - Func: runtimeNotify, Price: 1 << 15, RequiredFlags: callflag.AllowNotify}, -} - -func init() { - sort.Slice(defaultVMInterops, func(i, j int) bool { return defaultVMInterops[i].ID < defaultVMInterops[j].ID }) -} - -func defaultSyscallHandler(v *VM, id uint32) error { - n := sort.Search(len(defaultVMInterops), func(i int) bool { - return defaultVMInterops[i].ID >= id - }) - if n >= len(defaultVMInterops) || defaultVMInterops[n].ID != id { - return errors.New("syscall not found") - } - d := defaultVMInterops[n] - ctxFlag := v.Context().sc.callFlag - if !ctxFlag.Has(d.RequiredFlags) { - return fmt.Errorf("missing call flags: %05b vs %05b", ctxFlag, d.RequiredFlags) - } - return d.Func(v) -} - -// runtimeLog handles the syscall "System.Runtime.Log" for printing and logging stuff. -func runtimeLog(vm *VM) error { - msg := vm.Estack().Pop().String() - fmt.Printf("NEO-GO-VM (log) > %s\n", msg) - return nil -} - -// runtimeNotify handles the syscall "System.Runtime.Notify" for printing and logging stuff. -func runtimeNotify(vm *VM) error { - name := vm.Estack().Pop().String() - item := vm.Estack().Pop() - fmt.Printf("NEO-GO-VM (notify) > [%s] %s\n", name, item.Value()) - return nil -} - -// init sorts the global defaultVMInterops value. -func init() { - sort.Slice(defaultVMInterops, func(i, j int) bool { - return defaultVMInterops[i].ID < defaultVMInterops[j].ID - }) -} diff --git a/pkg/vm/vm.go b/pkg/vm/vm.go index 44e3f50d5..1e4f88df7 100644 --- a/pkg/vm/vm.go +++ b/pkg/vm/vm.go @@ -108,8 +108,6 @@ func NewWithTrigger(t trigger.Type) *VM { vm := &VM{ state: vmstate.None, trigger: t, - - SyscallHandler: defaultSyscallHandler, } initStack(&vm.istack, "invocation", nil) @@ -1458,6 +1456,9 @@ func (v *VM) execute(ctx *Context, op opcode.Opcode, parameter []byte) (err erro case opcode.SYSCALL: interopID := GetInteropID(parameter) + if v.SyscallHandler == nil { + panic("vm's SyscallHandler is not initialized") + } err := v.SyscallHandler(v, interopID) if err != nil { panic(fmt.Sprintf("failed to invoke syscall %d: %s", interopID, err)) diff --git a/pkg/vm/vm_test.go b/pkg/vm/vm_test.go index e3bd59256..48807040f 100644 --- a/pkg/vm/vm_test.go +++ b/pkg/vm/vm_test.go @@ -17,6 +17,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/bigint" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" @@ -2747,6 +2748,19 @@ func TestRemoveReferrer(t *testing.T) { assert.Equal(t, 0, int(vm.refs)) } +func TestUninitializedSyscallHandler(t *testing.T) { + v := newTestVM() + v.Reset(trigger.Application) // Reset SyscallHandler. + id := make([]byte, 4) + binary.LittleEndian.PutUint32(id, interopnames.ToID([]byte(interopnames.SystemRuntimeGasLeft))) + script := append([]byte{byte(opcode.SYSCALL)}, id...) + v.LoadScript(script) + err := v.Run() + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "SyscallHandler is not initialized"), err.Error()) + assert.Equal(t, true, v.HasFailed()) +} + func makeProgram(opcodes ...opcode.Opcode) []byte { prog := make([]byte, len(opcodes)+1) // RET for i := 0; i < len(opcodes); i++ { From f1ecdb82cc3a0a0bcf3f6b4bf38f2f1d06532ab9 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 4 Oct 2022 13:05:51 +0300 Subject: [PATCH 14/27] vm: add 'events' command to VM CLI And dump events automatically after HALTed or FAULTed end of execution. --- cli/vm/cli.go | 45 +++++++++++++++++++++++++++- cli/vm/cli_test.go | 58 ++++++++++++++++++++++++++++++++++++ internal/basicchain/basic.go | 20 +++++++++---- 3 files changed, 116 insertions(+), 7 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index b57c6c81c..58495944c 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -225,6 +225,12 @@ example: Description: "Dump opcodes of the current loaded program", Action: handleOps, }, + { + Name: "events", + Usage: "Dump events emitted by the current loaded program", + Description: "Dump events emitted by the current loaded program", + Action: handleEvents, + }, } var completer *readline.PrefixCompleter @@ -626,12 +632,17 @@ func runVMWithHandling(c *cli.Context) { writeErr(c.App.ErrWriter, err) } - var message string + var ( + message string + dumpNtf bool + ) switch { case v.HasFailed(): message = "" // the error will be printed on return + dumpNtf = true case v.HasHalted(): message = v.DumpEStack() + dumpNtf = true case v.AtBreakpoint(): ctx := v.Context() if ctx.NextIP() < ctx.LenInstr() { @@ -641,6 +652,16 @@ func runVMWithHandling(c *cli.Context) { message = "execution has finished" } } + if dumpNtf { + var e string + e, err = dumpEvents(c.App) + if err == nil && len(e) != 0 { + if message != "" { + message += "\n" + } + message += "Events:\n" + e + } + } if message != "" { fmt.Fprintln(c.App.Writer, message) } @@ -733,6 +754,28 @@ func changePrompt(app *cli.App) { } } +func handleEvents(c *cli.Context) error { + e, err := dumpEvents(c.App) + if err != nil { + writeErr(c.App.ErrWriter, err) + return nil + } + fmt.Fprintln(c.App.Writer, e) + return nil +} + +func dumpEvents(app *cli.App) (string, error) { + ic := getInteropContextFromContext(app) + if len(ic.Notifications) == 0 { + return "", nil + } + b, err := json.MarshalIndent(ic.Notifications, "", "\t") + if err != nil { + return "", fmt.Errorf("failed to marshal notifications: %w", err) + } + return string(b), nil +} + // Run waits for user input from Stdin and executes the passed command. func (c *VMCLI) Run() error { if getPrintLogoFromContext(c.shell) { diff --git a/cli/vm/cli_test.go b/cli/vm/cli_test.go index a84141930..4c71304fe 100644 --- a/cli/vm/cli_test.go +++ b/cli/vm/cli_test.go @@ -20,6 +20,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/compiler" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/core/interop/interopnames" + "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -158,6 +159,12 @@ func (e *executor) checkNextLine(t *testing.T, expected string) { require.Regexp(t, expected, line) } +func (e *executor) checkNextLineExact(t *testing.T, expected string) { + line, err := e.out.ReadString('\n') + require.NoError(t, err) + require.Equal(t, expected, line) +} + func (e *executor) checkError(t *testing.T, expectedErr error) { line, err := e.out.ReadString('\n') require.NoError(t, err) @@ -190,6 +197,30 @@ func (e *executor) checkStack(t *testing.T, items ...interface{}) { require.NoError(t, err) } +func (e *executor) checkEvents(t *testing.T, isKeywordExpected bool, events ...state.NotificationEvent) { + if isKeywordExpected { + e.checkNextLine(t, "Events:") + } + d := json.NewDecoder(e.out) + var actual interface{} + require.NoError(t, d.Decode(&actual)) + rawActual, err := json.Marshal(actual) + require.NoError(t, err) + + rawExpected, err := json.Marshal(events) + require.NoError(t, err) + require.JSONEq(t, string(rawExpected), string(rawActual)) + + // Decoder has it's own buffer, we need to return unread part to the output. + outRemain := e.out.String() + e.out.Reset() + _, err = gio.Copy(e.out, d.Buffered()) + require.NoError(t, err) + e.out.WriteString(outRemain) + _, err = e.out.ReadString('\n') + require.NoError(t, err) +} + func (e *executor) checkSlot(t *testing.T, items ...interface{}) { d := json.NewDecoder(e.out) var actual interface{} @@ -728,3 +759,30 @@ func TestRunWithState(t *testing.T) { e.checkNextLine(t, "READY: loaded 37 instructions") e.checkStack(t, 3) } + +func TestEvents(t *testing.T) { + e := newTestVMClIWithState(t) + + script := io.NewBufBinWriter() + h, err := e.cli.chain.GetContractScriptHash(2) // examples/runtime/runtime.go + require.NoError(t, err) + emit.AppCall(script.BinWriter, h, "notify", callflag.All, []interface{}{true, 5}) + e.runProg(t, + "loadhex "+hex.EncodeToString(script.Bytes()), + "run", + "events") + expectedEvent := state.NotificationEvent{ + ScriptHash: h, + Name: "Event", + Item: stackitem.NewArray([]stackitem.Item{ + stackitem.NewArray([]stackitem.Item{ + stackitem.Make(true), + stackitem.Make(5), + }), + }), + } + e.checkNextLine(t, "READY: loaded 44 instructions") + e.checkStack(t, stackitem.Null{}) + e.checkEvents(t, true, expectedEvent) // automatically printed after `run` command + e.checkEvents(t, false, expectedEvent) // printed after `events` command +} diff --git a/internal/basicchain/basic.go b/internal/basicchain/basic.go index 944008bfa..fc18d1153 100644 --- a/internal/basicchain/basic.go +++ b/internal/basicchain/basic.go @@ -23,7 +23,7 @@ import ( const neoAmount = 99999000 -// InitSimple initializes chain with single contract storing several storage values. +// InitSimple initializes chain with simple contracts from 'examples' folder. // It's not as complicated as chain got after Init and may be used for tests where // chain with a small amount of data is needed and for historical functionality testing. // Needs a path to the root directory. @@ -31,14 +31,19 @@ func InitSimple(t *testing.T, rootpath string, e *neotest.Executor) { // examplesPrefix is a prefix of the example smart-contracts. var examplesPrefix = filepath.Join(rootpath, "examples") + deployExample := func(t *testing.T, name string) util.Uint160 { + _, h := newDeployTx(t, e, e.Validator, + filepath.Join(examplesPrefix, name, name+".go"), + filepath.Join(examplesPrefix, name, name+".yml"), + true) + return h + } + // Block #1: deploy storage contract (examples/storage/storage.go). - _, storageHash := newDeployTx(t, e, e.Validator, - filepath.Join(examplesPrefix, "storage", "storage.go"), - filepath.Join(examplesPrefix, "storage", "storage.yml"), - true) + storageHash := deployExample(t, "storage") + storageValidatorInvoker := e.ValidatorInvoker(storageHash) // Block #2: put (1, 1) kv pair. - storageValidatorInvoker := e.ValidatorInvoker(storageHash) storageValidatorInvoker.Invoke(t, 1, "put", 1, 1) // Block #3: put (2, 2) kv pair. @@ -46,6 +51,9 @@ func InitSimple(t *testing.T, rootpath string, e *neotest.Executor) { // Block #4: update (1, 1) -> (1, 2). storageValidatorInvoker.Invoke(t, 1, "put", 1, 2) + + // Block #5: deploy runtime contract (examples/runtime/runtime.go). + _ = deployExample(t, "runtime") } // Init pushes some predefined set of transactions into the given chain, it needs a path to From 0036c89d63f54c60a5a4eb6d55aabd7d2b813c37 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 4 Oct 2022 14:19:42 +0300 Subject: [PATCH 15/27] vm: add 'env' command showing state of the blockchain-backed VM CLI A useful one. --- cli/vm/cli.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ cli/vm/cli_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index 58495944c..4b4845d99 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -39,6 +39,7 @@ import ( const ( chainKey = "chain" + chainCfgKey = "chainCfg" icKey = "ic" manifestKey = "manifest" exitFuncKey = "exitFunc" @@ -51,6 +52,11 @@ const ( stringType = "string" ) +// Various flag names. +const ( + verboseFlagFullName = "verbose" +) + var commands = []cli.Command{ { Name: "exit", @@ -231,6 +237,24 @@ example: Description: "Dump events emitted by the current loaded program", Action: handleEvents, }, + { + Name: "env", + Usage: "Dump state of the chain that is used for VM CLI invocations (use -v for verbose node configuration)", + UsageText: `env [-v]`, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: verboseFlagFullName + ",v", + Usage: "Print the whole blockchain node configuration.", + }, + }, + Description: `env [-v] + +Dump state of the chain that is used for VM CLI invocations (use -v for verbose node configuration). + +Example: +> env -v`, + Action: handleEnv, + }, } var completer *readline.PrefixCompleter @@ -318,6 +342,7 @@ func NewWithConfig(printLogotype bool, onExit func(int), c *readline.Config, cfg vmcli.shell.Metadata = map[string]interface{}{ chainKey: chain, + chainCfgKey: cfg, icKey: ic, manifestKey: new(manifest.Manifest), exitFuncKey: exitF, @@ -344,6 +369,10 @@ func getChainFromContext(app *cli.App) *core.Blockchain { return app.Metadata[chainKey].(*core.Blockchain) } +func getChainConfigFromContext(app *cli.App) config.Config { + return app.Metadata[chainCfgKey].(config.Config) +} + func getInteropContextFromContext(app *cli.App) *interop.Context { return app.Metadata[icKey].(*interop.Context) } @@ -764,6 +793,21 @@ func handleEvents(c *cli.Context) error { return nil } +func handleEnv(c *cli.Context) error { + bc := getChainFromContext(c.App) + cfg := getChainConfigFromContext(c.App) + message := fmt.Sprintf("Chain height: %d\nNetwork magic: %d\nDB type: %s\n", bc.BlockHeight(), bc.GetConfig().Magic, cfg.ApplicationConfiguration.DBConfiguration.Type) + if c.Bool(verboseFlagFullName) { + cfgBytes, err := json.MarshalIndent(cfg, "", "\t") + if err != nil { + return fmt.Errorf("failed to marshal node configuration: %w", err) + } + message += "Node config:\n" + string(cfgBytes) + "\n" + } + fmt.Fprint(c.App.Writer, message) + return nil +} + func dumpEvents(app *cli.App) (string, error) { ic := getInteropContextFromContext(app) if len(ic.Notifications) == 0 { diff --git a/cli/vm/cli_test.go b/cli/vm/cli_test.go index 4c71304fe..241e54e1a 100644 --- a/cli/vm/cli_test.go +++ b/cli/vm/cli_test.go @@ -786,3 +786,28 @@ func TestEvents(t *testing.T) { e.checkEvents(t, true, expectedEvent) // automatically printed after `run` command e.checkEvents(t, false, expectedEvent) // printed after `events` command } + +func TestEnv(t *testing.T) { + t.Run("default setup", func(t *testing.T) { + e := newTestVMCLI(t) + e.runProg(t, "env") + e.checkNextLine(t, "Chain height: 0") + e.checkNextLine(t, "Network magic: 42") + e.checkNextLine(t, "DB type: inmemory") + }) + t.Run("setup with state", func(t *testing.T) { + e := newTestVMClIWithState(t) + e.runProg(t, "env") + e.checkNextLine(t, "Chain height: 5") + e.checkNextLine(t, "Network magic: 42") + e.checkNextLine(t, "DB type: leveldb") + }) + t.Run("verbose", func(t *testing.T) { + e := newTestVMClIWithState(t) + e.runProg(t, "env -v") + e.checkNextLine(t, "Chain height: 5") + e.checkNextLine(t, "Network magic: 42") + e.checkNextLine(t, "DB type: leveldb") + e.checkNextLine(t, "Node config:") // Do not check exact node config. + }) +} From 79e13f73d86a0e0199f92b12b762fb6612e89292 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 6 Oct 2022 13:24:57 +0300 Subject: [PATCH 16/27] core, rpc: move getFakeNextBlock to Blockchain It's needed for VM CLI as far and may be improved later. --- cli/vm/cli.go | 46 ++++++++++---- internal/fakechain/fakechain.go | 2 +- pkg/core/blockchain.go | 31 ++++++++-- pkg/core/interop/contract/call_test.go | 6 +- pkg/core/interop/runtime/ext_test.go | 9 ++- pkg/core/interop/storage/storage_test.go | 3 +- pkg/core/native/invocation_test.go | 16 +++-- pkg/neotest/basic.go | 3 +- pkg/neotest/client.go | 7 ++- pkg/services/oracle/oracle.go | 2 +- pkg/services/oracle/response.go | 15 +++-- pkg/services/rpcsrv/client_test.go | 9 ++- pkg/services/rpcsrv/server.go | 76 ++++++++---------------- 13 files changed, 136 insertions(+), 89 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index 4b4845d99..320bac2b4 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -333,7 +333,10 @@ func NewWithConfig(printLogotype bool, onExit func(int), c *readline.Config, cfg return nil, cli.NewExitError(fmt.Errorf("could not initialize blockchain: %w", err), 1) } // Do not run chain, we need only state-related functionality from it. - ic := chain.GetTestVM(trigger.Application, nil, nil) + ic, err := chain.GetTestVM(trigger.Application, nil, nil) + if err != nil { + return nil, cli.NewExitError(fmt.Errorf("failed to create test VM: %w", err), 1) + } vmcli := VMCLI{ chain: chain, @@ -483,7 +486,10 @@ func handleSlots(c *cli.Context) error { } func handleLoadNEF(c *cli.Context) error { - resetState(c.App) + err := resetState(c.App) + if err != nil { + return err + } v := getVMFromContext(c.App) args := c.Args() if len(args) < 2 { @@ -503,7 +509,10 @@ func handleLoadNEF(c *cli.Context) error { } func handleLoadBase64(c *cli.Context) error { - resetState(c.App) + err := resetState(c.App) + if err != nil { + return err + } v := getVMFromContext(c.App) args := c.Args() if len(args) < 1 { @@ -520,7 +529,10 @@ func handleLoadBase64(c *cli.Context) error { } func handleLoadHex(c *cli.Context) error { - resetState(c.App) + err := resetState(c.App) + if err != nil { + return err + } v := getVMFromContext(c.App) args := c.Args() if len(args) < 1 { @@ -537,7 +549,10 @@ func handleLoadHex(c *cli.Context) error { } func handleLoadGo(c *cli.Context) error { - resetState(c.App) + err := resetState(c.App) + if err != nil { + return err + } v := getVMFromContext(c.App) args := c.Args() if len(args) < 1 { @@ -564,7 +579,10 @@ func handleLoadGo(c *cli.Context) error { } func handleReset(c *cli.Context) error { - resetState(c.App) + err := resetState(c.App) + if err != nil { + return err + } changePrompt(c.App) return nil } @@ -577,11 +595,15 @@ func finalizeInteropContext(app *cli.App) { // resetInteropContext calls finalizer for current interop context and replaces // it with the newly created one. -func resetInteropContext(app *cli.App) { +func resetInteropContext(app *cli.App) error { finalizeInteropContext(app) bc := getChainFromContext(app) - newIc := bc.GetTestVM(trigger.Application, nil, nil) + newIc, err := bc.GetTestVM(trigger.Application, nil, nil) + if err != nil { + return fmt.Errorf("failed to create test VM: %w", err) + } setInteropContextInContext(app, newIc) + return nil } // resetManifest removes manifest from app context. @@ -591,9 +613,13 @@ func resetManifest(app *cli.App) { // resetState resets state of the app (clear interop context and manifest) so that it's ready // to load new program. -func resetState(app *cli.App) { - resetInteropContext(app) +func resetState(app *cli.App) error { + err := resetInteropContext(app) + if err != nil { + return fmt.Errorf("failed to reset interop context state: %w", err) + } resetManifest(app) + return nil } func getManifestFromFile(name string) (*manifest.Manifest, error) { diff --git a/internal/fakechain/fakechain.go b/internal/fakechain/fakechain.go index 3ae130061..34c1f9b6c 100644 --- a/internal/fakechain/fakechain.go +++ b/internal/fakechain/fakechain.go @@ -298,7 +298,7 @@ func (chain *FakeChain) GetStorageItem(id int32, key []byte) state.StorageItem { } // GetTestVM implements the Blockchainer interface. -func (chain *FakeChain) GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *interop.Context { +func (chain *FakeChain) GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) (*interop.Context, error) { panic("TODO") } diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index ccdfb0524..7e79f9b06 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -2241,19 +2241,28 @@ func (bc *Blockchain) GetEnrollments() ([]state.Validator, error) { } // GetTestVM returns an interop context with VM set up for a test run. -func (bc *Blockchain) GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *interop.Context { +func (bc *Blockchain) GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) (*interop.Context, error) { + if b == nil { + var err error + h := bc.BlockHeight() + 1 + b, err = bc.getFakeNextBlock(h) + if err != nil { + return nil, fmt.Errorf("failed to create fake block for height %d: %w", h, err) + } + } systemInterop := bc.newInteropContext(t, bc.dao, b, tx) _ = systemInterop.SpawnVM() // All the other code suppose that the VM is ready. - return systemInterop + return systemInterop, nil } // GetTestHistoricVM returns an interop context with VM set up for a test run. -func (bc *Blockchain) GetTestHistoricVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) (*interop.Context, error) { +func (bc *Blockchain) GetTestHistoricVM(t trigger.Type, tx *transaction.Transaction, nextBlockHeight uint32) (*interop.Context, error) { if bc.config.KeepOnlyLatestState { return nil, errors.New("only latest state is supported") } - if b == nil { - return nil, errors.New("block is mandatory to produce test historic VM") + b, err := bc.getFakeNextBlock(nextBlockHeight) + if err != nil { + return nil, fmt.Errorf("failed to create fake block for height %d: %w", nextBlockHeight, err) } var mode = mpt.ModeAll if bc.config.RemoveUntraceableBlocks { @@ -2284,6 +2293,18 @@ func (bc *Blockchain) GetTestHistoricVM(t trigger.Type, tx *transaction.Transact return systemInterop, nil } +// getFakeNextBlock returns fake block with the specified index and pre-filled Timestamp field. +func (bc *Blockchain) getFakeNextBlock(nextBlockHeight uint32) (*block.Block, error) { + b := block.New(bc.config.StateRootInHeader) + b.Index = nextBlockHeight + hdr, err := bc.GetHeader(bc.GetHeaderHash(int(nextBlockHeight - 1))) + if err != nil { + return nil, err + } + b.Timestamp = hdr.Timestamp + uint64(bc.config.SecondsPerBlock*int(time.Second/time.Millisecond)) + return b, nil +} + // Various witness verification errors. var ( ErrWitnessHashMismatch = errors.New("witness hash mismatch") diff --git a/pkg/core/interop/contract/call_test.go b/pkg/core/interop/contract/call_test.go index f4159acd3..50648a4b6 100644 --- a/pkg/core/interop/contract/call_test.go +++ b/pkg/core/interop/contract/call_test.go @@ -32,7 +32,8 @@ var pathToInternalContracts = filepath.Join("..", "..", "..", "..", "internal", func TestGetCallFlags(t *testing.T) { bc, _ := chain.NewSingle(t) - ic := bc.GetTestVM(trigger.Application, &transaction.Transaction{}, &block.Block{}) + ic, err := bc.GetTestVM(trigger.Application, &transaction.Transaction{}, &block.Block{}) + require.NoError(t, err) ic.VM.LoadScriptWithHash([]byte{byte(opcode.RET)}, util.Uint160{1, 2, 3}, callflag.All) require.NoError(t, contract.GetCallFlags(ic)) @@ -41,7 +42,8 @@ func TestGetCallFlags(t *testing.T) { func TestCall(t *testing.T) { bc, _ := chain.NewSingle(t) - ic := bc.GetTestVM(trigger.Application, &transaction.Transaction{}, &block.Block{}) + ic, err := bc.GetTestVM(trigger.Application, &transaction.Transaction{}, &block.Block{}) + require.NoError(t, err) cs, currCs := contracts.GetTestContractState(t, pathToInternalContracts, 4, 5, random.Uint160()) // sender and IDs are not important for the test require.NoError(t, native.PutContractState(ic.DAO, cs)) diff --git a/pkg/core/interop/runtime/ext_test.go b/pkg/core/interop/runtime/ext_test.go index d5a878036..0b724379b 100644 --- a/pkg/core/interop/runtime/ext_test.go +++ b/pkg/core/interop/runtime/ext_test.go @@ -65,7 +65,8 @@ func getSharpTestGenesis(t *testing.T) *block.Block { func createVM(t testing.TB) (*vm.VM, *interop.Context, *core.Blockchain) { chain, _ := chain.NewSingle(t) - ic := chain.GetTestVM(trigger.Application, &transaction.Transaction{}, &block.Block{}) + ic, err := chain.GetTestVM(trigger.Application, &transaction.Transaction{}, &block.Block{}) + require.NoError(t, err) v := ic.SpawnVM() return v, ic, chain } @@ -523,7 +524,8 @@ func TestGetRandomCompatibility(t *testing.T) { b := getSharpTestGenesis(t) tx := getSharpTestTx(util.Uint160{}) - ic := bc.GetTestVM(trigger.Application, tx, b) + ic, err := bc.GetTestVM(trigger.Application, tx, b) + require.NoError(t, err) ic.Network = 860833102 // Old mainnet magic used by C# tests. ic.VM = vm.New() @@ -550,7 +552,8 @@ func TestNotify(t *testing.T) { caller := random.Uint160() newIC := func(name string, args interface{}) *interop.Context { _, _, bc, cs := getDeployedInternal(t) - ic := bc.GetTestVM(trigger.Application, nil, nil) + ic, err := bc.GetTestVM(trigger.Application, nil, nil) + require.NoError(t, err) ic.VM.LoadNEFMethod(&cs.NEF, caller, cs.Hash, callflag.NoneFlag, true, 0, -1, nil) ic.VM.Estack().PushVal(args) ic.VM.Estack().PushVal(name) diff --git a/pkg/core/interop/storage/storage_test.go b/pkg/core/interop/storage/storage_test.go index 7fa0f6f04..fec1820b2 100644 --- a/pkg/core/interop/storage/storage_test.go +++ b/pkg/core/interop/storage/storage_test.go @@ -300,7 +300,8 @@ func TestFind(t *testing.T) { func createVM(t testing.TB) (*vm.VM, *interop.Context, *core.Blockchain) { chain, _ := chain.NewSingle(t) - ic := chain.GetTestVM(trigger.Application, &transaction.Transaction{}, &block.Block{}) + ic, err := chain.GetTestVM(trigger.Application, &transaction.Transaction{}, &block.Block{}) + require.NoError(t, err) v := ic.SpawnVM() return v, ic, chain } diff --git a/pkg/core/native/invocation_test.go b/pkg/core/native/invocation_test.go index b6f55d9dc..519dd30da 100644 --- a/pkg/core/native/invocation_test.go +++ b/pkg/core/native/invocation_test.go @@ -74,7 +74,8 @@ func TestNativeContract_InvokeInternal(t *testing.T) { require.NotNil(t, md) t.Run("fail, bad current script hash", func(t *testing.T) { - ic := bc.GetTestVM(trigger.Application, nil, nil) + ic, err := bc.GetTestVM(trigger.Application, nil, nil) + require.NoError(t, err) v := ic.SpawnVM() fakeH := util.Uint160{1, 2, 3} v.LoadScriptWithHash(clState.NEF.Script, fakeH, callflag.All) @@ -83,7 +84,7 @@ func TestNativeContract_InvokeInternal(t *testing.T) { v.Context().Jump(md.Offset) // Bad current script hash - err := v.Run() + err = v.Run() require.Error(t, err) require.True(t, strings.Contains(err.Error(), fmt.Sprintf("native contract %s (version 0) not found", fakeH.StringLE())), err.Error()) }) @@ -104,7 +105,8 @@ func TestNativeContract_InvokeInternal(t *testing.T) { }) eBad := neotest.NewExecutor(t, bcBad, validatorBad, committeeBad) - ic := bcBad.GetTestVM(trigger.Application, nil, nil) + ic, err := bcBad.GetTestVM(trigger.Application, nil, nil) + require.NoError(t, err) v := ic.SpawnVM() v.LoadScriptWithHash(clState.NEF.Script, clState.Hash, callflag.All) // hash is not affected by native update history input := []byte{1, 2, 3, 4} @@ -112,13 +114,14 @@ func TestNativeContract_InvokeInternal(t *testing.T) { v.Context().Jump(md.Offset) // It's prohibited to call natives before NativeUpdateHistory[0] height. - err := v.Run() + err = v.Run() require.Error(t, err) require.True(t, strings.Contains(err.Error(), "native contract CryptoLib is active after height = 1")) // Add new block => CryptoLib should be active now. eBad.AddNewBlock(t) - ic = bcBad.GetTestVM(trigger.Application, nil, nil) + ic, err = bcBad.GetTestVM(trigger.Application, nil, nil) + require.NoError(t, err) v = ic.SpawnVM() v.LoadScriptWithHash(clState.NEF.Script, clState.Hash, callflag.All) // hash is not affected by native update history v.Estack().PushVal(input) @@ -130,7 +133,8 @@ func TestNativeContract_InvokeInternal(t *testing.T) { }) t.Run("success", func(t *testing.T) { - ic := bc.GetTestVM(trigger.Application, nil, nil) + ic, err := bc.GetTestVM(trigger.Application, nil, nil) + require.NoError(t, err) v := ic.SpawnVM() v.LoadScriptWithHash(clState.NEF.Script, clState.Hash, callflag.All) input := []byte{1, 2, 3, 4} diff --git a/pkg/neotest/basic.go b/pkg/neotest/basic.go index a115aa7a4..3ee1904ab 100644 --- a/pkg/neotest/basic.go +++ b/pkg/neotest/basic.go @@ -374,7 +374,8 @@ func TestInvoke(bc *core.Blockchain, tx *transaction.Transaction) (*vm.VM, error // `GetTestVM` as well as `Run` can use a transaction hash which will set a cached value. // This is unwanted behavior, so we explicitly copy the transaction to perform execution. ttx := *tx - ic := bc.GetTestVM(trigger.Application, &ttx, b) + ic, _ := bc.GetTestVM(trigger.Application, &ttx, b) + defer ic.Finalize() ic.VM.LoadWithFlags(tx.Script, callflag.All) diff --git a/pkg/neotest/client.go b/pkg/neotest/client.go index b9d676f2b..25ac8c158 100644 --- a/pkg/neotest/client.go +++ b/pkg/neotest/client.go @@ -51,11 +51,14 @@ func (e *Executor) ValidatorInvoker(h util.Uint160) *ContractInvoker { func (c *ContractInvoker) TestInvoke(t testing.TB, method string, args ...interface{}) (*vm.Stack, error) { tx := c.PrepareInvokeNoSign(t, method, args...) b := c.NewUnsignedBlock(t, tx) - ic := c.Chain.GetTestVM(trigger.Application, tx, b) + ic, err := c.Chain.GetTestVM(trigger.Application, tx, b) + if err != nil { + return nil, err + } t.Cleanup(ic.Finalize) ic.VM.LoadWithFlags(tx.Script, callflag.All) - err := ic.VM.Run() + err = ic.VM.Run() return ic.VM.Estack(), err } diff --git a/pkg/services/oracle/oracle.go b/pkg/services/oracle/oracle.go index 6dbb8f692..78322a4ba 100644 --- a/pkg/services/oracle/oracle.go +++ b/pkg/services/oracle/oracle.go @@ -29,7 +29,7 @@ type ( GetBaseExecFee() int64 GetConfig() config.ProtocolConfiguration GetMaxVerificationGAS() int64 - GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *interop.Context + GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) (*interop.Context, error) GetTransaction(util.Uint256) (*transaction.Transaction, uint32, error) } diff --git a/pkg/services/oracle/response.go b/pkg/services/oracle/response.go index d8523b2b2..f2d3667c0 100644 --- a/pkg/services/oracle/response.go +++ b/pkg/services/oracle/response.go @@ -3,6 +3,7 @@ package oracle import ( "encoding/hex" "errors" + "fmt" gio "io" "github.com/nspcc-dev/neo-go/pkg/core/fee" @@ -107,7 +108,10 @@ func (o *Oracle) CreateResponseTx(gasForResponse int64, vub uint32, resp *transa size := io.GetVarSize(tx) tx.Scripts = append(tx.Scripts, transaction.Witness{VerificationScript: oracleSignContract}) - gasConsumed, ok := o.testVerify(tx) + gasConsumed, ok, err := o.testVerify(tx) + if err != nil { + return nil, fmt.Errorf("failed to prepare `verify` invocation: %w", err) + } if !ok { return nil, errors.New("can't verify transaction") } @@ -131,18 +135,21 @@ func (o *Oracle) CreateResponseTx(gasForResponse int64, vub uint32, resp *transa return tx, nil } -func (o *Oracle) testVerify(tx *transaction.Transaction) (int64, bool) { +func (o *Oracle) testVerify(tx *transaction.Transaction) (int64, bool, error) { // (*Blockchain).GetTestVM calls Hash() method of the provided transaction; once being called, this // method caches transaction hash, but tx building is not yet completed and hash will be changed. // So, make a copy of the tx to avoid wrong hash caching. cp := *tx - ic := o.Chain.GetTestVM(trigger.Verification, &cp, nil) + ic, err := o.Chain.GetTestVM(trigger.Verification, &cp, nil) + if err != nil { + return 0, false, fmt.Errorf("failed to create test VM: %w", err) + } ic.VM.GasLimit = o.Chain.GetMaxVerificationGAS() ic.VM.LoadScriptWithHash(o.oracleScript, o.oracleHash, callflag.ReadOnly) ic.VM.Context().Jump(o.verifyOffset) ok := isVerifyOk(ic) - return ic.VM.GasConsumed(), ok + return ic.VM.GasConsumed(), ok, nil } func isVerifyOk(ic *interop.Context) bool { diff --git a/pkg/services/rpcsrv/client_test.go b/pkg/services/rpcsrv/client_test.go index aaf4b31f1..6ddb6885e 100644 --- a/pkg/services/rpcsrv/client_test.go +++ b/pkg/services/rpcsrv/client_test.go @@ -1181,7 +1181,8 @@ func TestCreateNEP17TransferTx(t *testing.T) { require.NoError(t, err) require.NoError(t, acc.SignTx(testchain.Network(), tx)) require.NoError(t, chain.VerifyTx(tx)) - ic := chain.GetTestVM(trigger.Application, tx, nil) + ic, err := chain.GetTestVM(trigger.Application, tx, nil) + require.NoError(t, err) ic.VM.LoadScriptWithFlags(tx.Script, callflag.All) require.NoError(t, ic.VM.Run()) }) @@ -1195,7 +1196,8 @@ func TestCreateNEP17TransferTx(t *testing.T) { }) require.NoError(t, err) require.NoError(t, chain.VerifyTx(tx)) - ic := chain.GetTestVM(trigger.Application, tx, nil) + ic, err := chain.GetTestVM(trigger.Application, tx, nil) + require.NoError(t, err) ic.VM.LoadScriptWithFlags(tx.Script, callflag.All) require.NoError(t, ic.VM.Run()) require.Equal(t, 2, len(ic.Notifications)) @@ -1228,7 +1230,8 @@ func TestCreateNEP17TransferTx(t *testing.T) { require.NoError(t, err) require.NoError(t, acc.SignTx(testchain.Network(), tx)) require.NoError(t, chain.VerifyTx(tx)) - ic := chain.GetTestVM(trigger.Application, tx, nil) + ic, err := chain.GetTestVM(trigger.Application, tx, nil) + require.NoError(t, err) ic.VM.LoadScriptWithFlags(tx.Script, callflag.All) require.NoError(t, ic.VM.Run()) }) diff --git a/pkg/services/rpcsrv/server.go b/pkg/services/rpcsrv/server.go index 15d24db4a..f3c0601b8 100644 --- a/pkg/services/rpcsrv/server.go +++ b/pkg/services/rpcsrv/server.go @@ -89,8 +89,8 @@ type ( GetNotaryServiceFeePerKey() int64 GetStateModule() core.StateRoot GetStorageItem(id int32, key []byte) state.StorageItem - GetTestHistoricVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) (*interop.Context, error) - GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *interop.Context + GetTestHistoricVM(t trigger.Type, tx *transaction.Transaction, nextBlockHeight uint32) (*interop.Context, error) + GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) (*interop.Context, error) GetTokenLastUpdated(acc util.Uint160) (map[int32]uint32, error) GetTransaction(util.Uint256) (*transaction.Transaction, uint32, error) GetValidators() ([]*keys.PublicKey, error) @@ -1065,11 +1065,10 @@ func (s *Server) invokeReadOnlyMulti(bw *io.BufBinWriter, h util.Uint160, method } script := bw.Bytes() tx := &transaction.Transaction{Script: script} - b, err := s.getFakeNextBlock(s.chain.BlockHeight() + 1) + ic, err := s.chain.GetTestVM(trigger.Application, tx, nil) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("faile to prepare test VM: %w", err) } - ic := s.chain.GetTestVM(trigger.Application, tx, b) ic.VM.GasLimit = core.HeaderVerificationGasLimit ic.VM.LoadScriptWithFlags(script, callflag.All) err = ic.VM.Run() @@ -1832,7 +1831,7 @@ func (s *Server) invokeFunction(reqParams params.Params) (interface{}, *neorpc.E // invokeFunctionHistoric implements the `invokeFunctionHistoric` RPC call. func (s *Server) invokeFunctionHistoric(reqParams params.Params) (interface{}, *neorpc.Error) { - b, respErr := s.getHistoricParams(reqParams) + nextH, respErr := s.getHistoricParams(reqParams) if respErr != nil { return nil, respErr } @@ -1843,7 +1842,7 @@ func (s *Server) invokeFunctionHistoric(reqParams params.Params) (interface{}, * if respErr != nil { return nil, respErr } - return s.runScriptInVM(trigger.Application, tx.Script, util.Uint160{}, tx, b, verbose) + return s.runScriptInVM(trigger.Application, tx.Script, util.Uint160{}, tx, &nextH, verbose) } func (s *Server) getInvokeFunctionParams(reqParams params.Params) (*transaction.Transaction, bool, *neorpc.Error) { @@ -1899,7 +1898,7 @@ func (s *Server) invokescript(reqParams params.Params) (interface{}, *neorpc.Err // invokescripthistoric implements the `invokescripthistoric` RPC call. func (s *Server) invokescripthistoric(reqParams params.Params) (interface{}, *neorpc.Error) { - b, respErr := s.getHistoricParams(reqParams) + nextH, respErr := s.getHistoricParams(reqParams) if respErr != nil { return nil, respErr } @@ -1910,7 +1909,7 @@ func (s *Server) invokescripthistoric(reqParams params.Params) (interface{}, *ne if respErr != nil { return nil, respErr } - return s.runScriptInVM(trigger.Application, tx.Script, util.Uint160{}, tx, b, verbose) + return s.runScriptInVM(trigger.Application, tx.Script, util.Uint160{}, tx, &nextH, verbose) } func (s *Server) getInvokeScriptParams(reqParams params.Params) (*transaction.Transaction, bool, *neorpc.Error) { @@ -1953,7 +1952,7 @@ func (s *Server) invokeContractVerify(reqParams params.Params) (interface{}, *ne // invokeContractVerifyHistoric implements the `invokecontractverifyhistoric` RPC call. func (s *Server) invokeContractVerifyHistoric(reqParams params.Params) (interface{}, *neorpc.Error) { - b, respErr := s.getHistoricParams(reqParams) + nextH, respErr := s.getHistoricParams(reqParams) if respErr != nil { return nil, respErr } @@ -1964,7 +1963,7 @@ func (s *Server) invokeContractVerifyHistoric(reqParams params.Params) (interfac if respErr != nil { return nil, respErr } - return s.runScriptInVM(trigger.Verification, invocationScript, scriptHash, tx, b, false) + return s.runScriptInVM(trigger.Verification, invocationScript, scriptHash, tx, &nextH, false) } func (s *Server) getInvokeContractVerifyParams(reqParams params.Params) (util.Uint160, *transaction.Transaction, []byte, *neorpc.Error) { @@ -2007,64 +2006,45 @@ func (s *Server) getInvokeContractVerifyParams(reqParams params.Params) (util.Ui // with the specified index to perform the historic call. It also checks that // specified stateroot is stored at the specified height for further request // handling consistency. -func (s *Server) getHistoricParams(reqParams params.Params) (*block.Block, *neorpc.Error) { +func (s *Server) getHistoricParams(reqParams params.Params) (uint32, *neorpc.Error) { if s.chain.GetConfig().KeepOnlyLatestState { - return nil, neorpc.NewInvalidRequestError(fmt.Sprintf("only latest state is supported: %s", errKeepOnlyLatestState)) + return 0, neorpc.NewInvalidRequestError(fmt.Sprintf("only latest state is supported: %s", errKeepOnlyLatestState)) } if len(reqParams) < 1 { - return nil, neorpc.ErrInvalidParams + return 0, neorpc.ErrInvalidParams } height, respErr := s.blockHeightFromParam(reqParams.Value(0)) if respErr != nil { hash, err := reqParams.Value(0).GetUint256() if err != nil { - return nil, neorpc.NewInvalidParamsError(fmt.Sprintf("invalid block hash or index or stateroot hash: %s", err)) + return 0, neorpc.NewInvalidParamsError(fmt.Sprintf("invalid block hash or index or stateroot hash: %s", err)) } b, err := s.chain.GetBlock(hash) if err != nil { stateH, err := s.chain.GetStateModule().GetLatestStateHeight(hash) if err != nil { - return nil, neorpc.NewInvalidParamsError(fmt.Sprintf("unknown block or stateroot: %s", err)) + return 0, neorpc.NewInvalidParamsError(fmt.Sprintf("unknown block or stateroot: %s", err)) } height = int(stateH) } else { height = int(b.Index) } } - b, err := s.getFakeNextBlock(uint32(height + 1)) - if err != nil { - return nil, neorpc.NewInternalServerError(fmt.Sprintf("can't create fake block for height %d: %s", height+1, err)) - } - return b, nil + return uint32(height) + 1, nil } -func (s *Server) getFakeNextBlock(nextBlockHeight uint32) (*block.Block, error) { - // When transferring funds, script execution does no auto GAS claim, - // because it depends on persisting tx height. - // This is why we provide block here. - b := block.New(s.stateRootEnabled) - b.Index = nextBlockHeight - hdr, err := s.chain.GetHeader(s.chain.GetHeaderHash(int(nextBlockHeight - 1))) - if err != nil { - return nil, err - } - b.Timestamp = hdr.Timestamp + uint64(s.chain.GetConfig().SecondsPerBlock*int(time.Second/time.Millisecond)) - return b, nil -} - -func (s *Server) prepareInvocationContext(t trigger.Type, script []byte, contractScriptHash util.Uint160, tx *transaction.Transaction, b *block.Block, verbose bool) (*interop.Context, *neorpc.Error) { +func (s *Server) prepareInvocationContext(t trigger.Type, script []byte, contractScriptHash util.Uint160, tx *transaction.Transaction, nextH *uint32, verbose bool) (*interop.Context, *neorpc.Error) { var ( err error ic *interop.Context ) - if b == nil { - b, err = s.getFakeNextBlock(s.chain.BlockHeight() + 1) + if nextH == nil { + ic, err = s.chain.GetTestVM(t, tx, nil) if err != nil { - return nil, neorpc.NewInternalServerError(fmt.Sprintf("can't create fake block: %s", err)) + return nil, neorpc.NewInternalServerError(fmt.Sprintf("failed to create test VM: %s", err)) } - ic = s.chain.GetTestVM(t, tx, b) } else { - ic, err = s.chain.GetTestHistoricVM(t, tx, b) + ic, err = s.chain.GetTestHistoricVM(t, tx, *nextH) if err != nil { return nil, neorpc.NewInternalServerError(fmt.Sprintf("failed to create historic VM: %s", err)) } @@ -2096,8 +2076,8 @@ func (s *Server) prepareInvocationContext(t trigger.Type, script []byte, contrac // witness invocation script in case of `verification` trigger (it pushes `verify` // arguments on stack before verification). In case of contract verification // contractScriptHash should be specified. -func (s *Server) runScriptInVM(t trigger.Type, script []byte, contractScriptHash util.Uint160, tx *transaction.Transaction, b *block.Block, verbose bool) (*result.Invoke, *neorpc.Error) { - ic, respErr := s.prepareInvocationContext(t, script, contractScriptHash, tx, b, verbose) +func (s *Server) runScriptInVM(t trigger.Type, script []byte, contractScriptHash util.Uint160, tx *transaction.Transaction, nextH *uint32, verbose bool) (*result.Invoke, *neorpc.Error) { + ic, respErr := s.prepareInvocationContext(t, script, contractScriptHash, tx, nextH, verbose) if respErr != nil { return nil, respErr } @@ -2111,16 +2091,12 @@ func (s *Server) runScriptInVM(t trigger.Type, script []byte, contractScriptHash var id uuid.UUID if sess != nil { - // b == nil only when we're not using MPT-backed storage, therefore + // nextH == nil only when we're not using MPT-backed storage, therefore // the second attempt won't stop here. - if s.config.SessionBackedByMPT && b == nil { + if s.config.SessionBackedByMPT && nextH == nil { ic.Finalize() - b, err = s.getFakeNextBlock(ic.Block.Index) - if err != nil { - return nil, neorpc.NewInternalServerError(fmt.Sprintf("unable to prepare block for historic call: %s", err)) - } // Rerun with MPT-backed storage. - return s.runScriptInVM(t, script, contractScriptHash, tx, b, verbose) + return s.runScriptInVM(t, script, contractScriptHash, tx, &ic.Block.Index, verbose) } id = uuid.New() sessionID := id.String() From ff03c33e6dade54a2cd7da7aef3922ae659a9911 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 4 Oct 2022 14:53:31 +0300 Subject: [PATCH 17/27] vm: allow historic calls inside VM CLI --- cli/vm/cli.go | 81 +++++++++++++++++++++++++++++++++------------- cli/vm/cli_test.go | 38 ++++++++++++++++++++++ 2 files changed, 97 insertions(+), 22 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index 320bac2b4..3b7734e21 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -54,9 +54,16 @@ const ( // Various flag names. const ( - verboseFlagFullName = "verbose" + verboseFlagFullName = "verbose" + historicFlagFullName = "historic" ) +var historicFlag = cli.IntFlag{ + Name: historicFlagFullName, + Usage: "Height for historic script invocation (for MPT-enabled blockchain configuration with KeepOnlyLatestState setting disabled). " + + "Assuming that block N-th is specified as an argument, the historic invocation is based on the storage state of height N and fake currently-accepting block with index N+1.", +} + var commands = []cli.Command{ { Name: "exit", @@ -113,7 +120,8 @@ var commands = []cli.Command{ Name: "loadnef", Usage: "Load a NEF-consistent script into the VM", UsageText: `loadnef `, - Description: `loadnef + Flags: []cli.Flag{historicFlag}, + Description: `loadnef [--historic ] both parameters are mandatory, example: > loadnef /path/to/script.nef /path/to/manifest.json`, Action: handleLoadNEF, @@ -121,8 +129,9 @@ both parameters are mandatory, example: { Name: "loadbase64", Usage: "Load a base64-encoded script string into the VM", - UsageText: `loadbase64 `, - Description: `loadbase64 + UsageText: `loadbase64 [--historic ] `, + Flags: []cli.Flag{historicFlag}, + Description: `loadbase64 [--historic ] is mandatory parameter, example: > loadbase64 AwAQpdToAAAADBQV9ehtQR1OrVZVhtHtoUHRfoE+agwUzmFvf3Rhfg/EuAVYOvJgKiON9j8TwAwIdHJhbnNmZXIMFDt9NxHG8Mz5sdypA9G/odiW8SOMQWJ9W1I4`, @@ -131,8 +140,9 @@ both parameters are mandatory, example: { Name: "loadhex", Usage: "Load a hex-encoded script string into the VM", - UsageText: `loadhex `, - Description: `loadhex + UsageText: `loadhex [--historic ] `, + Flags: []cli.Flag{historicFlag}, + Description: `loadhex [--historic ] is mandatory parameter, example: > loadhex 0c0c48656c6c6f20776f726c6421`, @@ -141,8 +151,9 @@ both parameters are mandatory, example: { Name: "loadgo", Usage: "Compile and load a Go file with the manifest into the VM", - UsageText: `loadgo `, - Description: `loadgo + UsageText: `loadgo [--historic ] `, + Flags: []cli.Flag{historicFlag}, + Description: `loadgo [--historic ] is mandatory parameter, example: > loadgo /path/to/file.go`, @@ -150,7 +161,8 @@ both parameters are mandatory, example: }, { Name: "reset", - Usage: "Unload compiled script from the VM", + Usage: "Unload compiled script from the VM and reset context to proper (possibly, historic) state", + Flags: []cli.Flag{historicFlag}, Action: handleReset, }, { @@ -485,8 +497,19 @@ func handleSlots(c *cli.Context) error { return nil } +// prepareVM retrieves --historic flag from context (if set) and resets app state +// (to the specified historic height if given). +func prepareVM(c *cli.Context) error { + if c.IsSet(historicFlagFullName) { + height := c.Int(historicFlagFullName) + return resetState(c.App, uint32(height)) + } + + return resetState(c.App) +} + func handleLoadNEF(c *cli.Context) error { - err := resetState(c.App) + err := prepareVM(c) if err != nil { return err } @@ -509,7 +532,7 @@ func handleLoadNEF(c *cli.Context) error { } func handleLoadBase64(c *cli.Context) error { - err := resetState(c.App) + err := prepareVM(c) if err != nil { return err } @@ -529,7 +552,7 @@ func handleLoadBase64(c *cli.Context) error { } func handleLoadHex(c *cli.Context) error { - err := resetState(c.App) + err := prepareVM(c) if err != nil { return err } @@ -549,7 +572,7 @@ func handleLoadHex(c *cli.Context) error { } func handleLoadGo(c *cli.Context) error { - err := resetState(c.App) + err := prepareVM(c) if err != nil { return err } @@ -579,7 +602,7 @@ func handleLoadGo(c *cli.Context) error { } func handleReset(c *cli.Context) error { - err := resetState(c.App) + err := prepareVM(c) if err != nil { return err } @@ -595,13 +618,25 @@ func finalizeInteropContext(app *cli.App) { // resetInteropContext calls finalizer for current interop context and replaces // it with the newly created one. -func resetInteropContext(app *cli.App) error { +func resetInteropContext(app *cli.App, height ...uint32) error { finalizeInteropContext(app) bc := getChainFromContext(app) - newIc, err := bc.GetTestVM(trigger.Application, nil, nil) - if err != nil { - return fmt.Errorf("failed to create test VM: %w", err) + var ( + newIc *interop.Context + err error + ) + if len(height) != 0 { + newIc, err = bc.GetTestHistoricVM(trigger.Application, nil, height[0]+1) + if err != nil { + return fmt.Errorf("failed to create historic VM for height %d: %w", height[0], err) + } + } else { + newIc, err = bc.GetTestVM(trigger.Application, nil, nil) + if err != nil { + return fmt.Errorf("failed to create VM: %w", err) + } } + setInteropContextInContext(app, newIc) return nil } @@ -613,10 +648,10 @@ func resetManifest(app *cli.App) { // resetState resets state of the app (clear interop context and manifest) so that it's ready // to load new program. -func resetState(app *cli.App) error { - err := resetInteropContext(app) +func resetState(app *cli.App, height ...uint32) error { + err := resetInteropContext(app, height...) if err != nil { - return fmt.Errorf("failed to reset interop context state: %w", err) + return err } resetManifest(app) return nil @@ -822,7 +857,9 @@ func handleEvents(c *cli.Context) error { func handleEnv(c *cli.Context) error { bc := getChainFromContext(c.App) cfg := getChainConfigFromContext(c.App) - message := fmt.Sprintf("Chain height: %d\nNetwork magic: %d\nDB type: %s\n", bc.BlockHeight(), bc.GetConfig().Magic, cfg.ApplicationConfiguration.DBConfiguration.Type) + ic := getInteropContextFromContext(c.App) + message := fmt.Sprintf("Chain height: %d\nVM height (may differ from chain height in case of historic call): %d\nNetwork magic: %d\nDB type: %s\n", + bc.BlockHeight(), ic.BlockHeight(), bc.GetConfig().Magic, cfg.ApplicationConfiguration.DBConfiguration.Type) if c.Bool(verboseFlagFullName) { cfgBytes, err := json.MarshalIndent(cfg, "", "\t") if err != nil { diff --git a/cli/vm/cli_test.go b/cli/vm/cli_test.go index 241e54e1a..1dc137cc1 100644 --- a/cli/vm/cli_test.go +++ b/cli/vm/cli_test.go @@ -760,6 +760,31 @@ func TestRunWithState(t *testing.T) { e.checkStack(t, 3) } +func TestRunWithHistoricState(t *testing.T) { + e := newTestVMClIWithState(t) + + script := io.NewBufBinWriter() + h, err := e.cli.chain.GetContractScriptHash(1) // examples/storage/storage.go + require.NoError(t, err) + emit.AppCall(script.BinWriter, h, "get", callflag.All, 1) + b := script.Bytes() + + e.runProg(t, + "loadhex "+hex.EncodeToString(b), // normal invocation + "run", + "loadhex --historic 3 "+hex.EncodeToString(b), // historic invocation, old value should be retrieved + "run", + "loadhex --historic 0 "+hex.EncodeToString(b), // historic invocation, contract is not deployed yet + "run", + ) + e.checkNextLine(t, "READY: loaded 36 instructions") + e.checkStack(t, []byte{2}) + e.checkNextLine(t, "READY: loaded 36 instructions") + e.checkStack(t, []byte{1}) + e.checkNextLine(t, "READY: loaded 36 instructions") + e.checkNextLineExact(t, "Error: at instruction 31 (SYSCALL): failed to invoke syscall 1381727586: called contract a00e3c2643a08a452d8b0bdd31849ae11a17c445 not found: key not found\n") +} + func TestEvents(t *testing.T) { e := newTestVMClIWithState(t) @@ -792,6 +817,7 @@ func TestEnv(t *testing.T) { e := newTestVMCLI(t) e.runProg(t, "env") e.checkNextLine(t, "Chain height: 0") + e.checkNextLineExact(t, "VM height (may differ from chain height in case of historic call): 0\n") e.checkNextLine(t, "Network magic: 42") e.checkNextLine(t, "DB type: inmemory") }) @@ -799,6 +825,17 @@ func TestEnv(t *testing.T) { e := newTestVMClIWithState(t) e.runProg(t, "env") e.checkNextLine(t, "Chain height: 5") + e.checkNextLineExact(t, "VM height (may differ from chain height in case of historic call): 5\n") + e.checkNextLine(t, "Network magic: 42") + e.checkNextLine(t, "DB type: leveldb") + }) + t.Run("setup with historic state", func(t *testing.T) { + e := newTestVMClIWithState(t) + e.runProg(t, "loadbase64 --historic 3 "+base64.StdEncoding.EncodeToString([]byte{byte(opcode.PUSH1)}), + "env") + e.checkNextLine(t, "READY: loaded 1 instructions") + e.checkNextLine(t, "Chain height: 5") + e.checkNextLineExact(t, "VM height (may differ from chain height in case of historic call): 3\n") e.checkNextLine(t, "Network magic: 42") e.checkNextLine(t, "DB type: leveldb") }) @@ -806,6 +843,7 @@ func TestEnv(t *testing.T) { e := newTestVMClIWithState(t) e.runProg(t, "env -v") e.checkNextLine(t, "Chain height: 5") + e.checkNextLineExact(t, "VM height (may differ from chain height in case of historic call): 5\n") e.checkNextLine(t, "Network magic: 42") e.checkNextLine(t, "DB type: leveldb") e.checkNextLine(t, "Node config:") // Do not check exact node config. From 8c781778067c2c2cebe36a8acde15c3f64054b9f Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 4 Oct 2022 15:38:42 +0300 Subject: [PATCH 18/27] vm: add 'storage' VM CLI command to dump storage items Another nice one, very useful for debugging. --- cli/vm/cli.go | 76 ++++++++++++++++++++++++++++++++++++++++++++-- cli/vm/cli_test.go | 31 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index 3b7734e21..fdcec3c2d 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -17,6 +17,7 @@ import ( "github.com/chzyer/readline" "github.com/kballard/go-shellquote" + "github.com/nspcc-dev/neo-go/cli/flags" "github.com/nspcc-dev/neo-go/pkg/compiler" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/core" @@ -54,8 +55,9 @@ const ( // Various flag names. const ( - verboseFlagFullName = "verbose" - historicFlagFullName = "historic" + verboseFlagFullName = "verbose" + historicFlagFullName = "historic" + backwardsFlagFullName = "backwards" ) var historicFlag = cli.IntFlag{ @@ -267,6 +269,32 @@ Example: > env -v`, Action: handleEnv, }, + { + Name: "storage", + Usage: "Dump storage of the contract with the specified hash, address or ID as is at the current stage of script invocation. " + + "Can be used if no script is loaded. " + + "Hex-encoded storage items prefix may be specified (empty by default to return the whole set of storage items). " + + "If seek prefix is not empty, then it's trimmed from the resulting keys." + + "Items are sorted. Backwards seek direction may be specified (false by default, which means forwards storage seek direction).", + UsageText: `storage [] [--backwards]`, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: backwardsFlagFullName + ",b", + Usage: "Backwards traversal direction", + }, + }, + Description: `storage --backwards + +Dump storage of the contract with the specified hash, address or ID as is at the current stage of script invocation. +Can be used if no script is loaded. +Hex-encoded storage items prefix may be specified (empty by default to return the whole set of storage items). +If seek prefix is not empty, then it's trimmed from the resulting keys. +Items are sorted. Backwards seek direction may be specified (false by default, which means forwards storage seek direction). + +Example: +> storage 0x0000000009070e030d0f0e020d0c06050e030c02 030e --backwards`, + Action: handleStorage, + }, } var completer *readline.PrefixCompleter @@ -871,6 +899,50 @@ func handleEnv(c *cli.Context) error { return nil } +func handleStorage(c *cli.Context) error { + if !c.Args().Present() { + return errors.New("contract hash, address or ID is mandatory argument") + } + hashOrID := c.Args().Get(0) + var ( + id int32 + ic = getInteropContextFromContext(c.App) + prefix []byte + backwards bool + ) + h, err := flags.ParseAddress(hashOrID) + if err != nil { + i, err := strconv.Atoi(hashOrID) + if err != nil { + return fmt.Errorf("failed to parse contract hash, address or ID: %w", err) + } + id = int32(i) + } else { + cs, err := ic.GetContract(h) + if err != nil { + return fmt.Errorf("contract %s not found: %w", h.StringLE(), err) + } + id = cs.ID + } + if c.NArg() > 1 { + prefix, err = hex.DecodeString(c.Args().Get(1)) + if err != nil { + return fmt.Errorf("failed to decode prefix from hex: %w", err) + } + } + if c.Bool(backwardsFlagFullName) { + backwards = true + } + ic.DAO.Seek(id, storage.SeekRange{ + Prefix: prefix, + Backwards: backwards, + }, func(k, v []byte) bool { + fmt.Fprintf(c.App.Writer, "%s: %v\n", hex.EncodeToString(k), hex.EncodeToString(v)) + return true + }) + return nil +} + func dumpEvents(app *cli.App) (string, error) { ic := getInteropContextFromContext(app) if len(ic.Notifications) == 0 { diff --git a/cli/vm/cli_test.go b/cli/vm/cli_test.go index 1dc137cc1..8c19b83b6 100644 --- a/cli/vm/cli_test.go +++ b/cli/vm/cli_test.go @@ -221,6 +221,12 @@ func (e *executor) checkEvents(t *testing.T, isKeywordExpected bool, events ...s require.NoError(t, err) } +func (e *executor) checkStorage(t *testing.T, kvs ...storage.KeyValue) { + for _, kv := range kvs { + e.checkNextLine(t, fmt.Sprintf("%s: %s", hex.EncodeToString(kv.Key), hex.EncodeToString(kv.Value))) + } +} + func (e *executor) checkSlot(t *testing.T, items ...interface{}) { d := json.NewDecoder(e.out) var actual interface{} @@ -849,3 +855,28 @@ func TestEnv(t *testing.T) { e.checkNextLine(t, "Node config:") // Do not check exact node config. }) } + +func TestDumpStorage(t *testing.T) { + e := newTestVMClIWithState(t) + + h, err := e.cli.chain.GetContractScriptHash(1) // examples/storage/storage.go + require.NoError(t, err) + expected := []storage.KeyValue{ + {Key: []byte{1}, Value: []byte{2}}, + {Key: []byte{2}, Value: []byte{2}}, + } + e.runProg(t, + "storage "+h.StringLE(), + "storage 0x"+h.StringLE(), + "storage "+address.Uint160ToString(h), + "storage 1", + "storage 1 "+hex.EncodeToString(expected[0].Key), + "storage 1 --backwards", + ) + e.checkStorage(t, expected...) + e.checkStorage(t, expected...) + e.checkStorage(t, expected...) + e.checkStorage(t, expected...) + e.checkStorage(t, storage.KeyValue{Key: nil, Value: []byte{2}}) // empty key because search prefix is trimmed + e.checkStorage(t, expected[1], expected[0]) +} From 0db4e8d62c15aa387a910d3a0e4bcc17fab982bf Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 5 Oct 2022 13:42:15 +0300 Subject: [PATCH 19/27] core: allow to perform storage search within given amount of DAO layers --- pkg/core/storage/memcached_store.go | 7 ++++++- pkg/core/storage/store.go | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/core/storage/memcached_store.go b/pkg/core/storage/memcached_store.go index 75e36dda5..f91c74fc4 100644 --- a/pkg/core/storage/memcached_store.go +++ b/pkg/core/storage/memcached_store.go @@ -301,7 +301,12 @@ func performSeek(ctx context.Context, ps Store, memRes []KeyValueExists, rng See } } } - ps.Seek(rng, mergeFunc) + if rng.SearchDepth == 0 || rng.SearchDepth > 1 { + if rng.SearchDepth > 1 { + rng.SearchDepth-- + } + ps.Seek(rng, mergeFunc) + } if !done && haveMem { loop: diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index caa8a4e73..0f2461bcd 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -59,6 +59,11 @@ type SeekRange struct { // whether seeking should be performed in a descending way. // Backwards can be safely combined with Prefix and Start. Backwards bool + // SearchDepth is the depth of Seek operation, denotes the number of cached + // DAO layers to perform search. Use 1 to fetch the latest changes from upper + // in-memory layer of cached DAO. Default 0 value denotes searching through + // the whole set of cached layers. + SearchDepth int } // ErrKeyNotFound is an error returned by Store implementations From cac4f6a4a6f153cb66d05bf015ccb980ec274aca Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 5 Oct 2022 13:56:26 +0300 Subject: [PATCH 20/27] cli: allow to dump storage diff for 'storage' VM CLI command --- cli/vm/cli.go | 24 ++++++++++++++++++------ cli/vm/cli_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index fdcec3c2d..a06c1ed08 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -58,6 +58,7 @@ const ( verboseFlagFullName = "verbose" historicFlagFullName = "historic" backwardsFlagFullName = "backwards" + diffFlagFullName = "diff" ) var historicFlag = cli.IntFlag{ @@ -275,24 +276,30 @@ Example: "Can be used if no script is loaded. " + "Hex-encoded storage items prefix may be specified (empty by default to return the whole set of storage items). " + "If seek prefix is not empty, then it's trimmed from the resulting keys." + - "Items are sorted. Backwards seek direction may be specified (false by default, which means forwards storage seek direction).", - UsageText: `storage [] [--backwards]`, + "Items are sorted. Backwards seek direction may be specified (false by default, which means forwards storage seek direction). " + + "It is possible to dump only those storage items that were added or changed during current script invocation (use --diff flag for it).", + UsageText: `storage [] [--backwards] [--diff]`, Flags: []cli.Flag{ cli.BoolFlag{ Name: backwardsFlagFullName + ",b", Usage: "Backwards traversal direction", }, + cli.BoolFlag{ + Name: diffFlagFullName + ",d", + Usage: "Dump only those storage items that were added or changed during the current script invocation. Note that this call won't show removed storage items.", + }, }, - Description: `storage --backwards + Description: `storage [--backwards] [--diff] Dump storage of the contract with the specified hash, address or ID as is at the current stage of script invocation. Can be used if no script is loaded. Hex-encoded storage items prefix may be specified (empty by default to return the whole set of storage items). If seek prefix is not empty, then it's trimmed from the resulting keys. Items are sorted. Backwards seek direction may be specified (false by default, which means forwards storage seek direction). +It is possible to dump only those storage items that were added or changed during current script invocation (use --diff flag for it). Example: -> storage 0x0000000009070e030d0f0e020d0c06050e030c02 030e --backwards`, +> storage 0x0000000009070e030d0f0e020d0c06050e030c02 030e --backwards --diff`, Action: handleStorage, }, } @@ -909,6 +916,7 @@ func handleStorage(c *cli.Context) error { ic = getInteropContextFromContext(c.App) prefix []byte backwards bool + seekDepth int ) h, err := flags.ParseAddress(hashOrID) if err != nil { @@ -933,9 +941,13 @@ func handleStorage(c *cli.Context) error { if c.Bool(backwardsFlagFullName) { backwards = true } + if c.Bool(diffFlagFullName) { + seekDepth = 1 // take only upper DAO layer which stores only added or updated items. + } ic.DAO.Seek(id, storage.SeekRange{ - Prefix: prefix, - Backwards: backwards, + Prefix: prefix, + Backwards: backwards, + SearchDepth: seekDepth, }, func(k, v []byte) bool { fmt.Fprintf(c.App.Writer, "%s: %v\n", hex.EncodeToString(k), hex.EncodeToString(v)) return true diff --git a/cli/vm/cli_test.go b/cli/vm/cli_test.go index 8c19b83b6..13df18241 100644 --- a/cli/vm/cli_test.go +++ b/cli/vm/cli_test.go @@ -880,3 +880,33 @@ func TestDumpStorage(t *testing.T) { e.checkStorage(t, storage.KeyValue{Key: nil, Value: []byte{2}}) // empty key because search prefix is trimmed e.checkStorage(t, expected[1], expected[0]) } + +func TestDumpStorageDiff(t *testing.T) { + e := newTestVMClIWithState(t) + + script := io.NewBufBinWriter() + h, err := e.cli.chain.GetContractScriptHash(1) // examples/storage/storage.go + require.NoError(t, err) + emit.AppCall(script.BinWriter, h, "put", callflag.All, 3, 3) + + expected := []storage.KeyValue{ + {Key: []byte{1}, Value: []byte{2}}, + {Key: []byte{2}, Value: []byte{2}}, + } + diff := storage.KeyValue{Key: []byte{3}, Value: []byte{3}} + e.runProg(t, + "storage 1", + "storage 1 --diff", + "loadhex "+hex.EncodeToString(script.Bytes()), + "run", + "storage 1", + "storage 1 --diff", + ) + + e.checkStorage(t, expected...) + // no script is executed => no diff + e.checkNextLine(t, "READY: loaded 37 instructions") + e.checkStack(t, 3) + e.checkStorage(t, append(expected, diff)...) + e.checkStorage(t, diff) +} From b3c8192d2e272ace4060add49c3b40b9959b6e39 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 5 Oct 2022 15:06:20 +0300 Subject: [PATCH 21/27] cli: add 'changes' command for VM CLI --- cli/vm/cli.go | 130 +++++++++++++++++++++++++++++++++++---------- cli/vm/cli_test.go | 68 ++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 28 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index a06c1ed08..3f5eb5e2e 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/elliptic" "encoding/base64" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -277,7 +278,8 @@ Example: "Hex-encoded storage items prefix may be specified (empty by default to return the whole set of storage items). " + "If seek prefix is not empty, then it's trimmed from the resulting keys." + "Items are sorted. Backwards seek direction may be specified (false by default, which means forwards storage seek direction). " + - "It is possible to dump only those storage items that were added or changed during current script invocation (use --diff flag for it).", + "It is possible to dump only those storage items that were added or changed during current script invocation (use --diff flag for it). " + + "To dump the whole set of storage changes including removed items use 'changes' command.", UsageText: `storage [] [--backwards] [--diff]`, Flags: []cli.Flag{ cli.BoolFlag{ @@ -286,7 +288,7 @@ Example: }, cli.BoolFlag{ Name: diffFlagFullName + ",d", - Usage: "Dump only those storage items that were added or changed during the current script invocation. Note that this call won't show removed storage items.", + Usage: "Dump only those storage items that were added or changed during the current script invocation. Note that this call won't show removed storage items, use 'changes' command for that.", }, }, Description: `storage [--backwards] [--diff] @@ -297,11 +299,32 @@ Hex-encoded storage items prefix may be specified (empty by default to return th If seek prefix is not empty, then it's trimmed from the resulting keys. Items are sorted. Backwards seek direction may be specified (false by default, which means forwards storage seek direction). It is possible to dump only those storage items that were added or changed during current script invocation (use --diff flag for it). +To dump the whole set of storage changes including removed items use 'changes' command. Example: > storage 0x0000000009070e030d0f0e020d0c06050e030c02 030e --backwards --diff`, Action: handleStorage, }, + { + Name: "changes", + Usage: "Dump storage changes as is at the current stage of loaded script invocation. " + + "If no script is loaded or executed, then no changes are present. " + + "The contract hash, address or ID may be specified as the first parameter to dump the specified contract storage changes. " + + "Hex-encoded search prefix (without contract ID) may be specified to dump matching storage changes. " + + "Resulting values are not sorted.", + UsageText: `changes [ []]`, + Description: `changes [ []] + +Dump storage changes as is at the current stage of loaded script invocation. +If no script is loaded or executed, then no changes are present. +The contract hash, address or ID may be specified as the first parameter to dump the specified contract storage changes. +Hex-encoded search prefix (without contract ID) may be specified to dump matching storage changes. +Resulting values are not sorted. + +Example: +> changes 0x0000000009070e030d0f0e020d0c06050e030c02 030e`, + Action: handleChanges, + }, } var completer *readline.PrefixCompleter @@ -907,37 +930,15 @@ func handleEnv(c *cli.Context) error { } func handleStorage(c *cli.Context) error { - if !c.Args().Present() { - return errors.New("contract hash, address or ID is mandatory argument") + id, prefix, err := getDumpArgs(c) + if err != nil { + return err } - hashOrID := c.Args().Get(0) var ( - id int32 - ic = getInteropContextFromContext(c.App) - prefix []byte backwards bool seekDepth int + ic = getInteropContextFromContext(c.App) ) - h, err := flags.ParseAddress(hashOrID) - if err != nil { - i, err := strconv.Atoi(hashOrID) - if err != nil { - return fmt.Errorf("failed to parse contract hash, address or ID: %w", err) - } - id = int32(i) - } else { - cs, err := ic.GetContract(h) - if err != nil { - return fmt.Errorf("contract %s not found: %w", h.StringLE(), err) - } - id = cs.ID - } - if c.NArg() > 1 { - prefix, err = hex.DecodeString(c.Args().Get(1)) - if err != nil { - return fmt.Errorf("failed to decode prefix from hex: %w", err) - } - } if c.Bool(backwardsFlagFullName) { backwards = true } @@ -955,6 +956,79 @@ func handleStorage(c *cli.Context) error { return nil } +func handleChanges(c *cli.Context) error { + var ( + expectedID int32 + prefix []byte + err error + hasAgs = c.Args().Present() + ) + if hasAgs { + expectedID, prefix, err = getDumpArgs(c) + if err != nil { + return err + } + } + ic := getInteropContextFromContext(c.App) + b := ic.DAO.GetBatch() + if b == nil { + return nil + } + ops := storage.BatchToOperations(b) + var notFirst bool + for _, op := range ops { + id := int32(binary.LittleEndian.Uint32(op.Key)) + if hasAgs && (expectedID != id || (len(prefix) != 0 && !bytes.HasPrefix(op.Key[4:], prefix))) { + continue + } + var message string + if notFirst { + message += "\n" + } + message += fmt.Sprintf("Contract ID: %d\nState: %s\nKey: %s\n", id, op.State, hex.EncodeToString(op.Key[4:])) + if op.Value != nil { + message += fmt.Sprintf("Value: %s\n", hex.EncodeToString(op.Value)) + } + fmt.Fprint(c.App.Writer, message) + notFirst = true + } + return nil +} + +// getDumpArgs is a helper function that retrieves contract ID and search prefix (if given). +func getDumpArgs(c *cli.Context) (int32, []byte, error) { + if !c.Args().Present() { + return 0, nil, errors.New("contract hash, address or ID is mandatory argument") + } + hashOrID := c.Args().Get(0) + var ( + ic = getInteropContextFromContext(c.App) + id int32 + prefix []byte + ) + h, err := flags.ParseAddress(hashOrID) + if err != nil { + i, err := strconv.Atoi(hashOrID) + if err != nil { + return 0, nil, fmt.Errorf("failed to parse contract hash, address or ID: %w", err) + } + id = int32(i) + } else { + cs, err := ic.GetContract(h) + if err != nil { + return 0, nil, fmt.Errorf("contract %s not found: %w", h.StringLE(), err) + } + id = cs.ID + } + if c.NArg() > 1 { + prefix, err = hex.DecodeString(c.Args().Get(1)) + if err != nil { + return 0, nil, fmt.Errorf("failed to decode prefix from hex: %w", err) + } + } + return id, prefix, nil +} + func dumpEvents(app *cli.App) (string, error) { ic := getInteropContextFromContext(app) if len(ic.Notifications) == 0 { diff --git a/cli/vm/cli_test.go b/cli/vm/cli_test.go index 13df18241..9fe83906d 100644 --- a/cli/vm/cli_test.go +++ b/cli/vm/cli_test.go @@ -23,6 +23,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" + "github.com/nspcc-dev/neo-go/pkg/core/storage/dboper" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/neotest" @@ -227,6 +228,20 @@ func (e *executor) checkStorage(t *testing.T, kvs ...storage.KeyValue) { } } +type storageChange struct { + ContractID int32 + dboper.Operation +} + +func (e *executor) checkChange(t *testing.T, c storageChange) { + e.checkNextLine(t, fmt.Sprintf("Contract ID: %d", c.ContractID)) + e.checkNextLine(t, fmt.Sprintf("State: %s", c.State)) + e.checkNextLine(t, fmt.Sprintf("Key: %s", hex.EncodeToString(c.Key))) + if c.Value != nil { + e.checkNextLine(t, fmt.Sprintf("Value: %s", hex.EncodeToString(c.Value))) + } +} + func (e *executor) checkSlot(t *testing.T, items ...interface{}) { d := json.NewDecoder(e.out) var actual interface{} @@ -910,3 +925,56 @@ func TestDumpStorageDiff(t *testing.T) { e.checkStorage(t, append(expected, diff)...) e.checkStorage(t, diff) } + +func TestDumpChanges(t *testing.T) { + e := newTestVMClIWithState(t) + + script := io.NewBufBinWriter() + h, err := e.cli.chain.GetContractScriptHash(1) // examples/storage/storage.go + require.NoError(t, err) + emit.AppCall(script.BinWriter, h, "put", callflag.All, 3, 4) // add + emit.AppCall(script.BinWriter, h, "delete", callflag.All, 1) // remove + emit.AppCall(script.BinWriter, h, "put", callflag.All, 2, 5) // update + + expected := []storageChange{ + { + ContractID: 1, + Operation: dboper.Operation{ + State: "Deleted", + Key: []byte{1}, + }, + }, + { + ContractID: 1, + Operation: dboper.Operation{ + State: "Changed", + Key: []byte{2}, + Value: []byte{5}, + }, + }, + { + ContractID: 1, + Operation: dboper.Operation{ + State: "Added", + Key: []byte{3}, + Value: []byte{4}, + }, + }, + } + e.runProg(t, + "changes", + "changes 1", + "loadhex "+hex.EncodeToString(script.Bytes()), + "run", + "changes 1 "+hex.EncodeToString([]byte{1}), + "changes 1 "+hex.EncodeToString([]byte{2}), + "changes 1 "+hex.EncodeToString([]byte{3}), + ) + + // no script is executed => no diff + e.checkNextLine(t, "READY: loaded 113 instructions") + e.checkStack(t, 3, true, 2) + e.checkChange(t, expected[0]) + e.checkChange(t, expected[1]) + e.checkChange(t, expected[2]) +} From b2cd007d8dd550cd6e61c00349021d73ae28eeb8 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Fri, 7 Oct 2022 14:51:51 +0300 Subject: [PATCH 22/27] cli: move handleLoggingParams to 'options' package It will be reused by other CLI packages. --- cli/options/options.go | 89 ++++++++++++++++++++++++++++++++++++++ cli/server/server.go | 91 ++------------------------------------- cli/server/server_test.go | 26 +++++------ 3 files changed, 102 insertions(+), 104 deletions(-) diff --git a/cli/options/options.go b/cli/options/options.go index b2bcedc13..7fe705992 100644 --- a/cli/options/options.go +++ b/cli/options/options.go @@ -6,16 +6,23 @@ package options import ( "context" "errors" + "fmt" + "net/url" + "os" + "runtime" "strconv" "time" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/config/netmode" "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/urfave/cli" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) // DefaultTimeout is the default timeout used for RPC requests. @@ -151,3 +158,85 @@ func GetConfigFromContext(ctx *cli.Context) (config.Config, error) { } return config.Load(configPath, GetNetwork(ctx)) } + +var ( + // _winfileSinkRegistered denotes whether zap has registered + // user-supplied factory for all sinks with `winfile`-prefixed scheme. + _winfileSinkRegistered bool + _winfileSinkCloser func() error +) + +// HandleLoggingParams reads logging parameters. +// If a user selected debug level -- function enables it. +// If logPath is configured -- function creates a dir and a file for logging. +// If logPath is configured on Windows -- function returns closer to be +// able to close sink for the opened log output file. +func HandleLoggingParams(debug bool, cfg config.ApplicationConfiguration) (*zap.Logger, func() error, error) { + level := zapcore.InfoLevel + if debug { + level = zapcore.DebugLevel + } + + cc := zap.NewProductionConfig() + cc.DisableCaller = true + cc.DisableStacktrace = true + cc.EncoderConfig.EncodeDuration = zapcore.StringDurationEncoder + cc.EncoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder + cc.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + cc.Encoding = "console" + cc.Level = zap.NewAtomicLevelAt(level) + cc.Sampling = nil + + if logPath := cfg.LogPath; logPath != "" { + if err := io.MakeDirForFile(logPath, "logger"); err != nil { + return nil, nil, err + } + + if runtime.GOOS == "windows" { + if !_winfileSinkRegistered { + // See https://github.com/uber-go/zap/issues/621. + err := zap.RegisterSink("winfile", func(u *url.URL) (zap.Sink, error) { + if u.User != nil { + return nil, fmt.Errorf("user and password not allowed with file URLs: got %v", u) + } + if u.Fragment != "" { + return nil, fmt.Errorf("fragments not allowed with file URLs: got %v", u) + } + if u.RawQuery != "" { + return nil, fmt.Errorf("query parameters not allowed with file URLs: got %v", u) + } + // Error messages are better if we check hostname and port separately. + if u.Port() != "" { + return nil, fmt.Errorf("ports not allowed with file URLs: got %v", u) + } + if hn := u.Hostname(); hn != "" && hn != "localhost" { + return nil, fmt.Errorf("file URLs must leave host empty or use localhost: got %v", u) + } + switch u.Path { + case "stdout": + return os.Stdout, nil + case "stderr": + return os.Stderr, nil + } + f, err := os.OpenFile(u.Path[1:], // Remove leading slash left after url.Parse. + os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + _winfileSinkCloser = func() error { + _winfileSinkCloser = nil + return f.Close() + } + return f, err + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to register windows-specific sinc: %w", err) + } + _winfileSinkRegistered = true + } + logPath = "winfile:///" + logPath + } + + cc.OutputPaths = []string{logPath} + } + + log, err := cc.Build() + return log, _winfileSinkCloser, err +} diff --git a/cli/server/server.go b/cli/server/server.go index 88d5ff790..827e3ae77 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -4,10 +4,8 @@ import ( "context" "errors" "fmt" - "net/url" "os" "os/signal" - "runtime" "syscall" "time" @@ -31,14 +29,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/services/stateroot" "github.com/urfave/cli" "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -var ( - // _winfileSinkRegistered denotes whether zap has registered - // user-supplied factory for all sinks with `winfile`-prefixed scheme. - _winfileSinkRegistered bool - _winfileSinkCloser func() error ) // NewCommands returns 'node' command. @@ -126,81 +116,6 @@ func newGraceContext() context.Context { return ctx } -// handleLoggingParams reads logging parameters. -// If a user selected debug level -- function enables it. -// If logPath is configured -- function creates a dir and a file for logging. -// If logPath is configured on Windows -- function returns closer to be -// able to close sink for the opened log output file. -func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) (*zap.Logger, func() error, error) { - level := zapcore.InfoLevel - if ctx.Bool("debug") { - level = zapcore.DebugLevel - } - - cc := zap.NewProductionConfig() - cc.DisableCaller = true - cc.DisableStacktrace = true - cc.EncoderConfig.EncodeDuration = zapcore.StringDurationEncoder - cc.EncoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder - cc.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - cc.Encoding = "console" - cc.Level = zap.NewAtomicLevelAt(level) - cc.Sampling = nil - - if logPath := cfg.LogPath; logPath != "" { - if err := io.MakeDirForFile(logPath, "logger"); err != nil { - return nil, nil, err - } - - if runtime.GOOS == "windows" { - if !_winfileSinkRegistered { - // See https://github.com/uber-go/zap/issues/621. - err := zap.RegisterSink("winfile", func(u *url.URL) (zap.Sink, error) { - if u.User != nil { - return nil, fmt.Errorf("user and password not allowed with file URLs: got %v", u) - } - if u.Fragment != "" { - return nil, fmt.Errorf("fragments not allowed with file URLs: got %v", u) - } - if u.RawQuery != "" { - return nil, fmt.Errorf("query parameters not allowed with file URLs: got %v", u) - } - // Error messages are better if we check hostname and port separately. - if u.Port() != "" { - return nil, fmt.Errorf("ports not allowed with file URLs: got %v", u) - } - if hn := u.Hostname(); hn != "" && hn != "localhost" { - return nil, fmt.Errorf("file URLs must leave host empty or use localhost: got %v", u) - } - switch u.Path { - case "stdout": - return os.Stdout, nil - case "stderr": - return os.Stderr, nil - } - f, err := os.OpenFile(u.Path[1:], // Remove leading slash left after url.Parse. - os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) - _winfileSinkCloser = func() error { - _winfileSinkCloser = nil - return f.Close() - } - return f, err - }) - if err != nil { - return nil, nil, fmt.Errorf("failed to register windows-specific sinc: %w", err) - } - _winfileSinkRegistered = true - } - logPath = "winfile:///" + logPath - } - - cc.OutputPaths = []string{logPath} - } - - log, err := cc.Build() - return log, _winfileSinkCloser, err -} - func initBCWithMetrics(cfg config.Config, log *zap.Logger) (*core.Blockchain, *metrics.Service, *metrics.Service, error) { chain, err := initBlockChain(cfg, log) if err != nil { @@ -225,7 +140,7 @@ func dumpDB(ctx *cli.Context) error { if err != nil { return cli.NewExitError(err, 1) } - log, logCloser, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) + log, logCloser, err := options.HandleLoggingParams(ctx.Bool("debug"), cfg.ApplicationConfiguration) if err != nil { return cli.NewExitError(err, 1) } @@ -278,7 +193,7 @@ func restoreDB(ctx *cli.Context) error { if err != nil { return err } - log, logCloser, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) + log, logCloser, err := options.HandleLoggingParams(ctx.Bool("debug"), cfg.ApplicationConfiguration) if err != nil { return cli.NewExitError(err, 1) } @@ -464,7 +379,7 @@ func startServer(ctx *cli.Context) error { if err != nil { return cli.NewExitError(err, 1) } - log, logCloser, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) + log, logCloser, err := options.HandleLoggingParams(ctx.Bool("debug"), cfg.ApplicationConfiguration) if err != nil { return cli.NewExitError(err, 1) } diff --git a/cli/server/server_test.go b/cli/server/server_test.go index 36d751cb1..d311db08e 100644 --- a/cli/server/server_test.go +++ b/cli/server/server_test.go @@ -7,13 +7,14 @@ import ( "path/filepath" "testing" + "go.uber.org/zap/zapcore" + "github.com/nspcc-dev/neo-go/cli/options" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/config/netmode" "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/stretchr/testify/require" "github.com/urfave/cli" - "go.uber.org/zap" "gopkg.in/yaml.v3" ) @@ -45,49 +46,42 @@ func TestHandleLoggingParams(t *testing.T) { t.Run("logdir is a file", func(t *testing.T) { logfile := filepath.Join(d, "logdir") require.NoError(t, os.WriteFile(logfile, []byte{1, 2, 3}, os.ModePerm)) - set := flag.NewFlagSet("flagSet", flag.ExitOnError) - ctx := cli.NewContext(cli.NewApp(), set, nil) cfg := config.ApplicationConfiguration{ LogPath: filepath.Join(logfile, "file.log"), } - _, closer, err := handleLoggingParams(ctx, cfg) + _, closer, err := options.HandleLoggingParams(false, cfg) require.Error(t, err) require.Nil(t, closer) }) t.Run("default", func(t *testing.T) { - set := flag.NewFlagSet("flagSet", flag.ExitOnError) - ctx := cli.NewContext(cli.NewApp(), set, nil) cfg := config.ApplicationConfiguration{ LogPath: testLog, } - logger, closer, err := handleLoggingParams(ctx, cfg) + logger, closer, err := options.HandleLoggingParams(false, cfg) require.NoError(t, err) t.Cleanup(func() { if closer != nil { require.NoError(t, closer()) } }) - require.True(t, logger.Core().Enabled(zap.InfoLevel)) - require.False(t, logger.Core().Enabled(zap.DebugLevel)) + require.True(t, logger.Core().Enabled(zapcore.InfoLevel)) + require.False(t, logger.Core().Enabled(zapcore.DebugLevel)) }) t.Run("debug", func(t *testing.T) { - set := flag.NewFlagSet("flagSet", flag.ExitOnError) - set.Bool("debug", true, "") - ctx := cli.NewContext(cli.NewApp(), set, nil) cfg := config.ApplicationConfiguration{ LogPath: testLog, } - logger, closer, err := handleLoggingParams(ctx, cfg) + logger, closer, err := options.HandleLoggingParams(true, cfg) require.NoError(t, err) t.Cleanup(func() { if closer != nil { require.NoError(t, closer()) } }) - require.True(t, logger.Core().Enabled(zap.InfoLevel)) - require.True(t, logger.Core().Enabled(zap.DebugLevel)) + require.True(t, logger.Core().Enabled(zapcore.InfoLevel)) + require.True(t, logger.Core().Enabled(zapcore.DebugLevel)) }) } @@ -104,7 +98,7 @@ func TestInitBCWithMetrics(t *testing.T) { ctx := cli.NewContext(cli.NewApp(), set, nil) cfg, err := options.GetConfigFromContext(ctx) require.NoError(t, err) - logger, closer, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) + logger, closer, err := options.HandleLoggingParams(true, cfg.ApplicationConfiguration) require.NoError(t, err) t.Cleanup(func() { if closer != nil { From 95cbddf19e7ddd2872f22b5e5a2d302cdfefa215 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Fri, 7 Oct 2022 15:27:24 +0300 Subject: [PATCH 23/27] cli: use custom logger to filter out runtime.Log messages ``` anna@kiwi:~/Documents/GitProjects/nspcc-dev/neo-go$ ./bin/neo-go vm -p _ ____________ __________ _ ____ ___ / | / / ____/ __ \ / ____/ __ \ | | / / |/ / / |/ / __/ / / / /_____/ / __/ / / /____| | / / /|_/ / / /| / /___/ /_/ /_____/ /_/ / /_/ /_____/ |/ / / / / /_/ |_/_____/\____/ \____/\____/ |___/_/ /_/ NEO-GO-VM > loadgo ./1-print/1-print.go READY: loaded 21 instructions NEO-GO-VM 0 > run 2022-10-07T15:28:20.461+0300 INFO runtime log {"tx": "", "script": "db03ceb3f672ee8cd0d714989b4d103ff7eed2f3", "msg": "Hello, world!"} [] ``` --- cli/options/filtering_core.go | 28 ++++++++++++++++++++++++++++ cli/vm/cli.go | 21 +++++++++++++++++++-- pkg/core/interop/runtime/engine.go | 5 ++++- 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 cli/options/filtering_core.go diff --git a/cli/options/filtering_core.go b/cli/options/filtering_core.go new file mode 100644 index 000000000..4dd0ad245 --- /dev/null +++ b/cli/options/filtering_core.go @@ -0,0 +1,28 @@ +package options + +import "go.uber.org/zap/zapcore" + +// FilteringCore is custom implementation of zapcore.Core that allows to filter +// log entries using custom filtering function. +type FilteringCore struct { + zapcore.Core + filter FilterFunc +} + +// FilterFunc is the filter function that is called to check whether the given +// entry together with the associated fields is to be written to a core or not. +type FilterFunc func(zapcore.Entry) bool + +// NewFilteringCore returns a core middleware that uses the given filter function +// to decide whether to log this message or not. +func NewFilteringCore(next zapcore.Core, filter FilterFunc) zapcore.Core { + return &FilteringCore{next, filter} +} + +// Check implements zapcore.Core interface and performs log entries filtering. +func (c *FilteringCore) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.filter(e) { + return c.Core.Check(e, ce) + } + return ce +} diff --git a/cli/vm/cli.go b/cli/vm/cli.go index 3f5eb5e2e..1a71dcfe3 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -19,10 +19,12 @@ import ( "github.com/chzyer/readline" "github.com/kballard/go-shellquote" "github.com/nspcc-dev/neo-go/cli/flags" + "github.com/nspcc-dev/neo-go/cli/options" "github.com/nspcc-dev/neo-go/pkg/compiler" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/core" "github.com/nspcc-dev/neo-go/pkg/core/interop" + "github.com/nspcc-dev/neo-go/pkg/core/interop/runtime" "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -37,6 +39,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/urfave/cli" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) const ( @@ -392,13 +395,27 @@ func NewWithConfig(printLogotype bool, onExit func(int), c *readline.Config, cfg store = storage.NewMemoryStore() } + log, logCloser, err := options.HandleLoggingParams(false, cfg.ApplicationConfiguration) + if err != nil { + return nil, cli.NewExitError(fmt.Errorf("failed to init logger: %w", err), 1) + } + filter := zap.WrapCore(func(z zapcore.Core) zapcore.Core { + return options.NewFilteringCore(z, func(entry zapcore.Entry) bool { + // Log only Runtime.Notify messages. + return entry.Level == zapcore.InfoLevel && entry.Message == runtime.SystemRuntimeLogMessage + }) + }) + fLog := log.WithOptions(filter) + exitF := func(i int) { _ = store.Close() + if logCloser != nil { + _ = logCloser() + } onExit(i) } - log := zap.NewNop() - chain, err := core.NewBlockchain(store, cfg.ProtocolConfiguration, log) + chain, err := core.NewBlockchain(store, cfg.ProtocolConfiguration, fLog) if err != nil { return nil, cli.NewExitError(fmt.Errorf("could not initialize blockchain: %w", err), 1) } diff --git a/pkg/core/interop/runtime/engine.go b/pkg/core/interop/runtime/engine.go index daadcda5a..42855c10a 100644 --- a/pkg/core/interop/runtime/engine.go +++ b/pkg/core/interop/runtime/engine.go @@ -19,6 +19,9 @@ const ( MaxEventNameLen = 32 // MaxNotificationSize is the maximum length of a runtime log message. MaxNotificationSize = 1024 + // SystemRuntimeLogMessage represents log entry message used for output + // of the System.Runtime.Log syscall. + SystemRuntimeLogMessage = "runtime log" ) // GetExecutingScriptHash returns executing script hash. @@ -112,7 +115,7 @@ func Log(ic *interop.Context) error { if ic.Tx != nil { txHash = ic.Tx.Hash().StringLE() } - ic.Log.Info("runtime log", + ic.Log.Info(SystemRuntimeLogMessage, zap.String("tx", txHash), zap.String("script", ic.VM.GetCurrentScriptHash().StringLE()), zap.String("msg", state)) From 16f5ae38120bd0b78ec886bab6bec38625c1c2cf Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Fri, 7 Oct 2022 15:31:07 +0300 Subject: [PATCH 24/27] cli: add upper bound check for contract ID for 'storage' VM CLI cmd --- cli/vm/cli.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index 1a71dcfe3..6c68ae988 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "math" "math/big" "os" "strconv" @@ -1029,6 +1030,9 @@ func getDumpArgs(c *cli.Context) (int32, []byte, error) { if err != nil { return 0, nil, fmt.Errorf("failed to parse contract hash, address or ID: %w", err) } + if i > math.MaxInt32 { + return 0, nil, fmt.Errorf("contract ID exceeds max int32 value") + } id = int32(i) } else { cs, err := ic.GetContract(h) From eac5e1526e37201935303ce3e79fcbc8b176cb5e Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Fri, 7 Oct 2022 15:47:21 +0300 Subject: [PATCH 25/27] cli: rename VMCLI to CLI --- cli/vm/cli.go | 12 ++++++------ cli/vm/cli_test.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index 6c68ae988..69456ea86 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -354,15 +354,15 @@ var ( ErrInvalidParameter = errors.New("can't parse argument") ) -// VMCLI object for interacting with the VM. -type VMCLI struct { +// CLI object for interacting with the VM. +type CLI struct { chain *core.Blockchain shell *cli.App } -// NewWithConfig returns new VMCLI instance using provided config and (optionally) +// NewWithConfig returns new CLI instance using provided config and (optionally) // provided node config for state-backed VM. -func NewWithConfig(printLogotype bool, onExit func(int), c *readline.Config, cfg config.Config) (*VMCLI, error) { +func NewWithConfig(printLogotype bool, onExit func(int), c *readline.Config, cfg config.Config) (*CLI, error) { if c.AutoComplete == nil { // Autocomplete commands/flags on TAB. c.AutoComplete = completer @@ -426,7 +426,7 @@ func NewWithConfig(printLogotype bool, onExit func(int), c *readline.Config, cfg return nil, cli.NewExitError(fmt.Errorf("failed to create test VM: %w", err), 1) } - vmcli := VMCLI{ + vmcli := CLI{ chain: chain, shell: ctl, } @@ -1063,7 +1063,7 @@ func dumpEvents(app *cli.App) (string, error) { } // Run waits for user input from Stdin and executes the passed command. -func (c *VMCLI) Run() error { +func (c *CLI) Run() error { if getPrintLogoFromContext(c.shell) { printLogo(c.shell.Writer) } diff --git a/cli/vm/cli_test.go b/cli/vm/cli_test.go index 9fe83906d..7bfe9d54a 100644 --- a/cli/vm/cli_test.go +++ b/cli/vm/cli_test.go @@ -62,7 +62,7 @@ func (r *readCloser) WriteString(s string) { type executor struct { in *readCloser out *bytes.Buffer - cli *VMCLI + cli *CLI ch chan struct{} exit atomic.Bool } @@ -126,7 +126,7 @@ func newTestVMClIWithState(t *testing.T) *executor { basicchain.InitSimple(t, "../../", e) bc.Close() - // After that create VMCLI backed by created chain. + // After that create CLI backed by created chain. configPath := "../../config/protocol.unit_testnet.yml" cfg, err := config.LoadFile(configPath) require.NoError(t, err) From 735db08f8449c90b357cb016c92af470eb9a1cd1 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Fri, 7 Oct 2022 16:06:12 +0300 Subject: [PATCH 26/27] services: adjust RPC server's getHistoricParams Update documentation and add index upper bound check to get rid of CodeQL warning. --- pkg/services/rpcsrv/server.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/services/rpcsrv/server.go b/pkg/services/rpcsrv/server.go index f3c0601b8..fbc19bd90 100644 --- a/pkg/services/rpcsrv/server.go +++ b/pkg/services/rpcsrv/server.go @@ -2002,8 +2002,8 @@ func (s *Server) getInvokeContractVerifyParams(reqParams params.Params) (util.Ui return scriptHash, tx, invocationScript, nil } -// getHistoricParams checks that historic calls are supported and returns fake block -// with the specified index to perform the historic call. It also checks that +// getHistoricParams checks that historic calls are supported and returns index of +// a fake next block to perform the historic call. It also checks that // specified stateroot is stored at the specified height for further request // handling consistency. func (s *Server) getHistoricParams(reqParams params.Params) (uint32, *neorpc.Error) { @@ -2030,6 +2030,9 @@ func (s *Server) getHistoricParams(reqParams params.Params) (uint32, *neorpc.Err height = int(b.Index) } } + if height > math.MaxUint32 { + return 0, neorpc.NewInvalidParamsError("historic height exceeds max uint32 value") + } return uint32(height) + 1, nil } From 4d2afa262430a98571b29a0eae065c8462c65f6a Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 7 Oct 2022 17:10:04 +0300 Subject: [PATCH 27/27] cli/vm: use ParseInt to properly (and easily) check for int32 --- cli/vm/cli.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cli/vm/cli.go b/cli/vm/cli.go index 69456ea86..ad4f3a184 100644 --- a/cli/vm/cli.go +++ b/cli/vm/cli.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "io" - "math" "math/big" "os" "strconv" @@ -1026,13 +1025,10 @@ func getDumpArgs(c *cli.Context) (int32, []byte, error) { ) h, err := flags.ParseAddress(hashOrID) if err != nil { - i, err := strconv.Atoi(hashOrID) + i, err := strconv.ParseInt(hashOrID, 10, 32) if err != nil { return 0, nil, fmt.Errorf("failed to parse contract hash, address or ID: %w", err) } - if i > math.MaxInt32 { - return 0, nil, fmt.Errorf("contract ID exceeds max int32 value") - } id = int32(i) } else { cs, err := ic.GetContract(h)