From e7e91ef63407d4946f70ffa746e33d79cfea1999 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 16:31:20 +0300 Subject: [PATCH] [#1689] adm: Remove storagecfg subcommand It is unused and unsupported for a long time. Change-Id: I570567db4e8cb202e41286064406ad85cd0e7a39 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/root.go | 2 - .../internal/modules/storagecfg/config.go | 135 ------ .../internal/modules/storagecfg/root.go | 432 ------------------ 3 files changed, 569 deletions(-) delete mode 100644 cmd/frostfs-adm/internal/modules/storagecfg/config.go delete mode 100644 cmd/frostfs-adm/internal/modules/storagecfg/root.go diff --git a/cmd/frostfs-adm/internal/modules/root.go b/cmd/frostfs-adm/internal/modules/root.go index defd898c8..e42204b7a 100644 --- a/cmd/frostfs-adm/internal/modules/root.go +++ b/cmd/frostfs-adm/internal/modules/root.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/storagecfg" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/autocomplete" utilConfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" @@ -41,7 +40,6 @@ func init() { rootCmd.AddCommand(config.RootCmd) rootCmd.AddCommand(morph.RootCmd) - rootCmd.AddCommand(storagecfg.RootCmd) rootCmd.AddCommand(metabase.RootCmd) rootCmd.AddCommand(autocomplete.Command("frostfs-adm")) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/config.go b/cmd/frostfs-adm/internal/modules/storagecfg/config.go deleted file mode 100644 index 67e3414c2..000000000 --- a/cmd/frostfs-adm/internal/modules/storagecfg/config.go +++ /dev/null @@ -1,135 +0,0 @@ -package storagecfg - -const configTemplate = `logger: - level: info # logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" - -node: - wallet: - path: {{ .Wallet.Path }} # path to a NEO wallet; ignored if key is presented - address: {{ .Wallet.Account }} # address of a NEO account in the wallet; ignored if key is presented - password: {{ .Wallet.Password }} # password for a NEO account in the wallet; ignored if key is presented - addresses: # list of addresses announced by Storage node in the Network map - - {{ .AnnouncedAddress }} - attribute_0: UN-LOCODE:{{ .Attribute.Locode }} - relay: {{ .Relay }} # start Storage node in relay mode without bootstrapping into the Network map - -grpc: - num: 1 # total number of listener endpoints - 0: - endpoint: {{ .Endpoint }} # endpoint for gRPC server - tls:{{if .TLSCert}} - enabled: true # enable TLS for a gRPC connection (min version is TLS 1.2) - certificate: {{ .TLSCert }} # path to TLS certificate - key: {{ .TLSKey }} # path to TLS key - {{- else }} - enabled: false # disable TLS for a gRPC connection - {{- end}} - -control: - authorized_keys: # list of hex-encoded public keys that have rights to use the Control Service - {{- range .AuthorizedKeys }} - - {{.}}{{end}} - grpc: - endpoint: {{.ControlEndpoint}} # endpoint that is listened by the Control Service - -morph: - dial_timeout: 20s # timeout for side chain NEO RPC client connection - cache_ttl: 15s # use TTL cache for side chain GET operations - rpc_endpoint: # side chain N3 RPC endpoints - {{- range .MorphRPC }} - - address: wss://{{.}}/ws{{end}} -{{if not .Relay }} -storage: - shard: - default: # section with the default shard parameters - metabase: - perm: 0644 # permissions for metabase files(directories: +x for current user and group) - - blobstor: - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) - depth: 2 # max depth of object tree storage in FS - small_object_size: 102400 # 100KiB, size threshold for "small" objects which are stored in key-value DB, not in FS, bytes - compress: true # turn on/off Zstandard compression (level 3) of stored objects - compression_exclude_content_types: - - audio/* - - video/* - - blobovnicza: - size: 1073741824 # approximate size limit of single blobovnicza instance, total size will be: size*width^(depth+1), bytes - depth: 1 # max depth of object tree storage in key-value DB - width: 4 # max width of object tree storage in key-value DB - opened_cache_capacity: 50 # maximum number of opened database files - opened_cache_ttl: 5m # ttl for opened database file - opened_cache_exp_interval: 15s # cache cleanup interval for expired blobovnicza's - - gc: - remover_batch_size: 200 # number of objects to be removed by the garbage collector - remover_sleep_interval: 5m # frequency of the garbage collector invocation - 0: - mode: "read-write" # mode of the shard, must be one of the: "read-write" (default), "read-only" - - metabase: - path: {{ .MetabasePath }} # path to the metabase - - blobstor: - path: {{ .BlobstorPath }} # path to the blobstor -{{end}}` - -const ( - neofsMainnetAddress = "2cafa46838e8b564468ebd868dcafdd99dce6221" - balanceMainnetAddress = "dc1ec98d9d0c5f9dfade16144defe08cffc5ca55" - neofsTestnetAddress = "b65d8243ac63983206d17e5221af0653a7266fa1" - balanceTestnetAddress = "e0420c216003747626670d1424569c17c79015bf" -) - -var n3config = map[string]struct { - MorphRPC []string - RPC []string - NeoFSContract string - BalanceContract string -}{ - "testnet": { - MorphRPC: []string{ - "rpc01.morph.testnet.fs.neo.org:51331", - "rpc02.morph.testnet.fs.neo.org:51331", - "rpc03.morph.testnet.fs.neo.org:51331", - "rpc04.morph.testnet.fs.neo.org:51331", - "rpc05.morph.testnet.fs.neo.org:51331", - "rpc06.morph.testnet.fs.neo.org:51331", - "rpc07.morph.testnet.fs.neo.org:51331", - }, - RPC: []string{ - "rpc01.testnet.n3.nspcc.ru:21331", - "rpc02.testnet.n3.nspcc.ru:21331", - "rpc03.testnet.n3.nspcc.ru:21331", - "rpc04.testnet.n3.nspcc.ru:21331", - "rpc05.testnet.n3.nspcc.ru:21331", - "rpc06.testnet.n3.nspcc.ru:21331", - "rpc07.testnet.n3.nspcc.ru:21331", - }, - NeoFSContract: neofsTestnetAddress, - BalanceContract: balanceTestnetAddress, - }, - "mainnet": { - MorphRPC: []string{ - "rpc1.morph.fs.neo.org:40341", - "rpc2.morph.fs.neo.org:40341", - "rpc3.morph.fs.neo.org:40341", - "rpc4.morph.fs.neo.org:40341", - "rpc5.morph.fs.neo.org:40341", - "rpc6.morph.fs.neo.org:40341", - "rpc7.morph.fs.neo.org:40341", - }, - RPC: []string{ - "rpc1.n3.nspcc.ru:10331", - "rpc2.n3.nspcc.ru:10331", - "rpc3.n3.nspcc.ru:10331", - "rpc4.n3.nspcc.ru:10331", - "rpc5.n3.nspcc.ru:10331", - "rpc6.n3.nspcc.ru:10331", - "rpc7.n3.nspcc.ru:10331", - }, - NeoFSContract: neofsMainnetAddress, - BalanceContract: balanceMainnetAddress, - }, -} diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/root.go b/cmd/frostfs-adm/internal/modules/storagecfg/root.go deleted file mode 100644 index a5adea0da..000000000 --- a/cmd/frostfs-adm/internal/modules/storagecfg/root.go +++ /dev/null @@ -1,432 +0,0 @@ -package storagecfg - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "fmt" - "math/rand" - "net" - "net/url" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "text/template" - "time" - - netutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" - "github.com/chzyer/readline" - "github.com/nspcc-dev/neo-go/cli/flags" - "github.com/nspcc-dev/neo-go/cli/input" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/encoding/address" - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" - "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" - "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/spf13/cobra" -) - -const ( - walletFlag = "wallet" - accountFlag = "account" -) - -const ( - defaultControlEndpoint = "localhost:8090" - defaultDataEndpoint = "localhost" -) - -// RootCmd is a root command of config section. -var RootCmd = &cobra.Command{ - Use: "storage-config [-w wallet] [-a acccount] []", - Short: "Section for storage node configuration commands", - Run: storageConfig, -} - -func init() { - fs := RootCmd.Flags() - - fs.StringP(walletFlag, "w", "", "Path to wallet") - fs.StringP(accountFlag, "a", "", "Wallet account") -} - -type config struct { - AnnouncedAddress string - AuthorizedKeys []string - ControlEndpoint string - Endpoint string - TLSCert string - TLSKey string - MorphRPC []string - Attribute struct { - Locode string - } - Wallet struct { - Path string - Account string - Password string - } - Relay bool - BlobstorPath string - MetabasePath string -} - -func storageConfig(cmd *cobra.Command, args []string) { - outPath := getOutputPath(args) - - historyPath := filepath.Join(os.TempDir(), "frostfs-adm.history") - readline.SetHistoryPath(historyPath) - - var c config - - c.Wallet.Path, _ = cmd.Flags().GetString(walletFlag) - if c.Wallet.Path == "" { - c.Wallet.Path = getPath("Path to the storage node wallet: ") - } - - w, err := wallet.NewWalletFromFile(c.Wallet.Path) - fatalOnErr(err) - - fillWalletAccount(cmd, &c, w) - - accH, err := flags.ParseAddress(c.Wallet.Account) - fatalOnErr(err) - - acc := w.GetAccount(accH) - if acc == nil { - fatalOnErr(errors.New("can't find account in wallet")) - } - - c.Wallet.Password, err = input.ReadPassword(fmt.Sprintf("Enter password for %s > ", c.Wallet.Account)) - fatalOnErr(err) - - err = acc.Decrypt(c.Wallet.Password, keys.NEP2ScryptParams()) - fatalOnErr(err) - - c.AuthorizedKeys = append(c.AuthorizedKeys, hex.EncodeToString(acc.PrivateKey().PublicKey().Bytes())) - - network := readNetwork(cmd) - - c.MorphRPC = n3config[network].MorphRPC - - depositGas(cmd, acc, network) - - c.Attribute.Locode = getString("UN-LOCODE attribute in [XX YYY] format: ") - - endpoint := getDefaultEndpoint(cmd, &c) - c.Endpoint = getString(fmt.Sprintf("Listening address [%s]: ", endpoint)) - if c.Endpoint == "" { - c.Endpoint = endpoint - } - - c.ControlEndpoint = getString(fmt.Sprintf("Listening address (control endpoint) [%s]: ", defaultControlEndpoint)) - if c.ControlEndpoint == "" { - c.ControlEndpoint = defaultControlEndpoint - } - - c.TLSCert = getPath("TLS Certificate (optional): ") - if c.TLSCert != "" { - c.TLSKey = getPath("TLS Key: ") - } - - c.Relay = getConfirmation(false, "Use node as a relay? yes/[no]: ") - if !c.Relay { - p := getPath("Path to the storage directory (all available storage will be used): ") - c.BlobstorPath = filepath.Join(p, "blob") - c.MetabasePath = filepath.Join(p, "meta") - } - - out := applyTemplate(c) - fatalOnErr(os.WriteFile(outPath, out, 0o644)) - - cmd.Println("Node is ready for work! Run `frostfs-node -config " + outPath + "`") -} - -func getDefaultEndpoint(cmd *cobra.Command, c *config) string { - var addr, port string - for { - c.AnnouncedAddress = getString("Publicly announced address: ") - validator := netutil.Address{} - err := validator.FromString(c.AnnouncedAddress) - if err != nil { - cmd.Println("Incorrect address format. See https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/pkg/network/address.go for details.") - continue - } - uriAddr, err := url.Parse(validator.URIAddr()) - if err != nil { - panic(fmt.Errorf("unexpected error: %w", err)) - } - addr = uriAddr.Hostname() - port = uriAddr.Port() - ip, err := net.ResolveIPAddr("ip", addr) - if err != nil { - cmd.Printf("Can't resolve IP address %s: %v\n", addr, err) - continue - } - - if !ip.IP.IsGlobalUnicast() { - cmd.Println("IP must be global unicast.") - continue - } - cmd.Printf("Resolved IP address: %s\n", ip.String()) - - _, err = strconv.ParseUint(port, 10, 16) - if err != nil { - cmd.Println("Port must be an integer.") - continue - } - - break - } - return net.JoinHostPort(defaultDataEndpoint, port) -} - -func fillWalletAccount(cmd *cobra.Command, c *config, w *wallet.Wallet) { - c.Wallet.Account, _ = cmd.Flags().GetString(accountFlag) - if c.Wallet.Account == "" { - addr := address.Uint160ToString(w.GetChangeAddress()) - c.Wallet.Account = getWalletAccount(w, fmt.Sprintf("Wallet account [%s]: ", addr)) - if c.Wallet.Account == "" { - c.Wallet.Account = addr - } - } -} - -func readNetwork(cmd *cobra.Command) string { - var network string - for { - network = getString("Choose network [mainnet]/testnet: ") - switch network { - case "": - network = "mainnet" - case "testnet", "mainnet": - default: - cmd.Println(`Network must be either "mainnet" or "testnet"`) - continue - } - break - } - return network -} - -func getOutputPath(args []string) string { - if len(args) != 0 { - return args[0] - } - outPath := getPath("File to write config at [./config.yml]: ") - if outPath == "" { - outPath = "./config.yml" - } - return outPath -} - -func getWalletAccount(w *wallet.Wallet, prompt string) string { - addrs := make([]readline.PrefixCompleterInterface, len(w.Accounts)) - for i := range w.Accounts { - addrs[i] = readline.PcItem(w.Accounts[i].Address) - } - - readline.SetAutoComplete(readline.NewPrefixCompleter(addrs...)) - defer readline.SetAutoComplete(nil) - - s, err := readline.Line(prompt) - fatalOnErr(err) - return strings.TrimSpace(s) // autocompleter can return a string with a trailing space -} - -func getString(prompt string) string { - s, err := readline.Line(prompt) - fatalOnErr(err) - if s != "" { - _ = readline.AddHistory(s) - } - return s -} - -type filenameCompleter struct{} - -func (filenameCompleter) Do(line []rune, pos int) (newLine [][]rune, length int) { - prefix := string(line[:pos]) - dir := filepath.Dir(prefix) - de, err := os.ReadDir(dir) - if err != nil { - return nil, 0 - } - - for i := range de { - name := filepath.Join(dir, de[i].Name()) - if strings.HasPrefix(name, prefix) { - tail := []rune(strings.TrimPrefix(name, prefix)) - if de[i].IsDir() { - tail = append(tail, filepath.Separator) - } - newLine = append(newLine, tail) - } - } - if pos != 0 { - return newLine, pos - len([]rune(dir)) - } - return newLine, 0 -} - -func getPath(prompt string) string { - readline.SetAutoComplete(filenameCompleter{}) - defer readline.SetAutoComplete(nil) - - p, err := readline.Line(prompt) - fatalOnErr(err) - - if p == "" { - return p - } - - _ = readline.AddHistory(p) - - abs, err := filepath.Abs(p) - if err != nil { - fatalOnErr(fmt.Errorf("can't create an absolute path: %w", err)) - } - - return abs -} - -func getConfirmation(def bool, prompt string) bool { - for { - s, err := readline.Line(prompt) - fatalOnErr(err) - - switch strings.ToLower(s) { - case "y", "yes": - return true - case "n", "no": - return false - default: - if len(s) == 0 { - return def - } - } - } -} - -func applyTemplate(c config) []byte { - tmpl, err := template.New("config").Parse(configTemplate) - fatalOnErr(err) - - b := bytes.NewBuffer(nil) - fatalOnErr(tmpl.Execute(b, c)) - - return b.Bytes() -} - -func fatalOnErr(err error) { - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func depositGas(cmd *cobra.Command, acc *wallet.Account, network string) { - sideClient := initClient(n3config[network].MorphRPC) - balanceHash, _ := util.Uint160DecodeStringLE(n3config[network].BalanceContract) - - sideActor, err := actor.NewSimple(sideClient, acc) - if err != nil { - fatalOnErr(fmt.Errorf("creating actor over side chain client: %w", err)) - } - - sideGas := nep17.NewReader(sideActor, balanceHash) - accSH := acc.Contract.ScriptHash() - - balance, err := sideGas.BalanceOf(accSH) - if err != nil { - fatalOnErr(fmt.Errorf("side chain balance: %w", err)) - } - - ok := getConfirmation(false, fmt.Sprintf("Current NeoFS balance is %s, make a deposit? y/[n]: ", - fixedn.ToString(balance, 12))) - if !ok { - return - } - - amountStr := getString("Enter amount in GAS: ") - amount, err := fixedn.FromString(amountStr, 8) - if err != nil { - fatalOnErr(fmt.Errorf("invalid amount: %w", err)) - } - - mainClient := initClient(n3config[network].RPC) - neofsHash, _ := util.Uint160DecodeStringLE(n3config[network].NeoFSContract) - - mainActor, err := actor.NewSimple(mainClient, acc) - if err != nil { - fatalOnErr(fmt.Errorf("creating actor over main chain client: %w", err)) - } - - mainGas := nep17.New(mainActor, gas.Hash) - - txHash, _, err := mainGas.Transfer(accSH, neofsHash, amount, nil) - if err != nil { - fatalOnErr(fmt.Errorf("sending TX to the NeoFS contract: %w", err)) - } - - cmd.Print("Waiting for transactions to persist.") - tick := time.NewTicker(time.Second / 2) - defer tick.Stop() - - timer := time.NewTimer(time.Second * 20) - defer timer.Stop() - - at := trigger.Application - -loop: - for { - select { - case <-tick.C: - _, err := mainClient.GetApplicationLog(txHash, &at) - if err == nil { - cmd.Print("\n") - break loop - } - cmd.Print(".") - case <-timer.C: - cmd.Printf("\nTimeout while waiting for transaction to persist.\n") - if getConfirmation(false, "Continue configuration? yes/[no]: ") { - return - } - os.Exit(1) - } - } -} - -func initClient(rpc []string) *rpcclient.Client { - var c *rpcclient.Client - var err error - - shuffled := slices.Clone(rpc) - rand.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] }) - - for _, endpoint := range shuffled { - c, err = rpcclient.New(context.Background(), "https://"+endpoint, rpcclient.Options{ - DialTimeout: time.Second * 2, - RequestTimeout: time.Second * 5, - }) - if err != nil { - continue - } - if err = c.Init(); err != nil { - continue - } - return c - } - - fatalOnErr(fmt.Errorf("can't create N3 client: %w", err)) - panic("unreachable") -}