From 17f7adb640a18fef40c835acfaa225f26e3da481 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 8 Apr 2024 12:27:30 +0300 Subject: [PATCH] [#1065] adm: Add support EC parameters Signed-off-by: Alexander Chuprov --- cmd/frostfs-adm/docs/deploy.md | 2 ++ cmd/frostfs-adm/internal/commonflags/flags.go | 2 ++ .../internal/modules/config/config.go | 6 ++++ .../internal/modules/config/config_test.go | 2 ++ .../internal/modules/morph/config/config.go | 29 +++++++++++++++---- .../internal/modules/morph/helper/netmap.go | 2 ++ .../internal/modules/morph/initialize/root.go | 2 ++ dev/adm/frostfs-adm.yml | 2 ++ pkg/morph/client/netmap/config.go | 2 ++ 9 files changed, 43 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index 3b9f4c98..e510f478 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -34,6 +34,8 @@ alphabet-wallets: /home/user/deploy/alphabet-wallets network: max_object_size: 67108864 epoch_duration: 240 + max_ec_data_count: 12 + max_ec_parity_count: 4 fee: candidate: 0 container: 0 diff --git a/cmd/frostfs-adm/internal/commonflags/flags.go b/cmd/frostfs-adm/internal/commonflags/flags.go index bf6c17bd..81395edb 100644 --- a/cmd/frostfs-adm/internal/commonflags/flags.go +++ b/cmd/frostfs-adm/internal/commonflags/flags.go @@ -26,6 +26,8 @@ const ( ContractsURLFlagDesc = "URL to archive with compiled FrostFS contracts" EpochDurationInitFlag = "network.epoch_duration" MaxObjectSizeInitFlag = "network.max_object_size" + MaxECDataCountFlag = "network.max_ec_data_count" + MaxECParityCounFlag = "network.max_ec_parity_count" RefillGasAmountFlag = "gas" StorageWalletFlag = "storage-wallet" ContainerFeeInitFlag = "network.fee.container" diff --git a/cmd/frostfs-adm/internal/modules/config/config.go b/cmd/frostfs-adm/internal/modules/config/config.go index dd98a98b..a98245d0 100644 --- a/cmd/frostfs-adm/internal/modules/config/config.go +++ b/cmd/frostfs-adm/internal/modules/config/config.go @@ -21,6 +21,8 @@ type configTemplate struct { CandidateFee int ContainerFee int ContainerAliasFee int + MaxECDataCount int + MaxECParityCount int WithdrawFee int Glagolitics []string HomomorphicHashDisabled bool @@ -31,6 +33,8 @@ alphabet-wallets: {{ .AlphabetDir}} network: max_object_size: {{ .MaxObjectSize}} epoch_duration: {{ .EpochDuration}} + max_ec_data_count: {{ .MaxECDataCount}} + max_ec_parity_count: {{ .MaxECParityCount}} homomorphic_hash_disabled: {{ .HomomorphicHashDisabled}} fee: candidate: {{ .CandidateFee}} @@ -106,6 +110,8 @@ func generateConfigExample(appDir string, credSize int) (string, error) { tmpl := configTemplate{ Endpoint: "https://neo.rpc.node:30333", MaxObjectSize: 67108864, // 64 MiB + MaxECDataCount: 12, // Tested with 16-node networks, assuming 12 data + 4 parity nodes. + MaxECParityCount: 4, // Maximum 4 parity chunks, typically <= 3 for most policies. EpochDuration: 240, // 1 hour with 15s per block HomomorphicHashDisabled: false, // object homomorphic hash is enabled CandidateFee: 100_0000_0000, // 100.0 GAS (Fixed8) diff --git a/cmd/frostfs-adm/internal/modules/config/config_test.go b/cmd/frostfs-adm/internal/modules/config/config_test.go index 3fa800a0..beb1210e 100644 --- a/cmd/frostfs-adm/internal/modules/config/config_test.go +++ b/cmd/frostfs-adm/internal/modules/config/config_test.go @@ -27,6 +27,8 @@ func TestGenerateConfigExample(t *testing.T) { require.Equal(t, "https://neo.rpc.node:30333", v.GetString("rpc-endpoint")) require.Equal(t, filepath.Join(appDir, "alphabet-wallets"), v.GetString("alphabet-wallets")) require.Equal(t, 67108864, v.GetInt("network.max_object_size")) + require.Equal(t, 12, v.GetInt("network.max_ec_data_count")) + require.Equal(t, 4, v.GetInt("network.max_ec_parity_count")) require.Equal(t, 240, v.GetInt("network.epoch_duration")) require.Equal(t, 10000000000, v.GetInt("network.fee.candidate")) require.Equal(t, 1000, v.GetInt("network.fee.container")) diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index 2238a38d..f35e7aa0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -60,7 +60,8 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { switch k { case netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, - netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: + netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig, + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig: nbuf := make([]byte, 8) copy(nbuf[:], v) n := binary.LittleEndian.Uint64(nbuf) @@ -103,10 +104,8 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { } forceFlag, _ := cmd.Flags().GetBool(forceConfigSet) - bw := io.NewBufBinWriter() prm := make(map[string]any) - for _, arg := range args { k, v, err := parseConfigPair(arg, forceFlag) if err != nil { @@ -116,7 +115,7 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { prm[k] = v } - if err := validateConfig(prm); err != nil { + if err := validateConfig(prm, forceFlag); err != nil { return err } @@ -138,12 +137,29 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { return wCtx.AwaitTx() } -func validateConfig(args map[string]any) error { +func validateConfig(args map[string]any, forceFlag bool) error { + var sumEC int64 + _, okData := args[netmap.MaxECDataCountConfig] + _, okParity := args[netmap.MaxECParityCountConfig] + if okData != okParity { + return fmt.Errorf("both %s and %s must be present in the configuration", + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig) + } + for k, v := range args { value, ok := v.(int64) if !ok || value < 0 { return fmt.Errorf("%s must be >= 0, got %v", k, v) } + + if k == netmap.MaxECDataCountConfig || k == netmap.MaxECParityCountConfig { + sumEC += value + } + } + + if sumEC > 256 && !forceFlag { + return fmt.Errorf("the sum of %s and %s must be <= 256, got %d", + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig, sumEC) } return nil } @@ -160,7 +176,8 @@ func parseConfigPair(kvStr string, force bool) (key string, val any, err error) switch key { case netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, - netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: + netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig, + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig: val, err = strconv.ParseInt(valRaw, 10, 64) if err != nil { err = fmt.Errorf("could not parse %s's value '%s' as int: %w", key, valRaw, err) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go index f68fbefa..9ea27d3a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go @@ -35,6 +35,8 @@ func GetDefaultNetmapContractConfigMap() map[string]any { m := make(map[string]any) m[netmap.EpochDurationConfig] = viper.GetInt64(commonflags.EpochDurationInitFlag) m[netmap.MaxObjectSizeConfig] = viper.GetInt64(commonflags.MaxObjectSizeInitFlag) + m[netmap.MaxECDataCountConfig] = viper.GetInt64(commonflags.MaxECDataCountFlag) + m[netmap.MaxECParityCountConfig] = viper.GetInt64(commonflags.MaxECParityCounFlag) m[netmap.ContainerFeeConfig] = viper.GetInt64(commonflags.ContainerFeeInitFlag) m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(commonflags.ContainerAliasFeeInitFlag) m[netmap.IrCandidateFeeConfig] = viper.GetInt64(commonflags.CandidateFeeInitFlag) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go index 581bef53..b7885c51 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go @@ -25,6 +25,8 @@ var Cmd = &cobra.Command{ _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.EpochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) _ = viper.BindPFlag(commonflags.MaxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) + _ = viper.BindPFlag(commonflags.MaxECDataCountFlag, cmd.Flags().Lookup(commonflags.MaxECDataCountFlag)) + _ = viper.BindPFlag(commonflags.MaxECParityCounFlag, cmd.Flags().Lookup(commonflags.MaxECParityCounFlag)) _ = viper.BindPFlag(commonflags.HomomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) _ = viper.BindPFlag(commonflags.CandidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) _ = viper.BindPFlag(commonflags.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) diff --git a/dev/adm/frostfs-adm.yml b/dev/adm/frostfs-adm.yml index 5dc87e70..e75cc27f 100644 --- a/dev/adm/frostfs-adm.yml +++ b/dev/adm/frostfs-adm.yml @@ -6,6 +6,8 @@ network: basic_income_rate: 100000000 homomorphic_hash_disabled: false maintenance_mode_allowed: true + max_ec_data_count: 12 + max_ec_parity_count: 4 fee: audit: 10000 candidate: 10000000000 diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 21ba145d..2d19a819 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -11,6 +11,8 @@ import ( const ( MaxObjectSizeConfig = "MaxObjectSize" + MaxECParityCountConfig = "MaxECParityCount" + MaxECDataCountConfig = "MaxECDataCount" EpochDurationConfig = "EpochDuration" ContainerFeeConfig = "ContainerFee" ContainerAliasFeeConfig = "ContainerAliasFee"