diff --git a/cmd/s3-lifecycler/app.go b/cmd/s3-lifecycler/app.go index dc7cd9f..7cee56c 100644 --- a/cmd/s3-lifecycler/app.go +++ b/cmd/s3-lifecycler/app.go @@ -5,17 +5,21 @@ import ( "fmt" "os" "os/signal" + "sync" "syscall" - "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/credential/walletsource" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/frostfs" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/lifecycle" "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/morph" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/morph/contract" "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/notificator" "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/viper" "go.uber.org/zap" @@ -25,11 +29,18 @@ type ( App struct { log *zap.Logger logLevel zap.AtomicLevel + key *keys.PrivateKey cfg *viper.Viper done chan struct{} appServices []*metrics.Service appMetrics *metrics.AppMetrics notificator *notificator.Notificator + settings *appSettings + } + + appSettings struct { + mu sync.RWMutex + serviceKeys []*keys.PublicKey } ) @@ -47,6 +58,7 @@ func newApp(ctx context.Context, cfg *viper.Viper, log *Logger) *App { cfg: cfg, done: make(chan struct{}), appMetrics: metrics.NewAppMetrics(), + settings: newAppSettings(cfg, log), } a.appMetrics.SetHealth(HealthStatusStarting) @@ -56,76 +68,144 @@ func newApp(ctx context.Context, cfg *viper.Viper, log *Logger) *App { } func (a *App) init(ctx context.Context) { - key, err := fetchKey(a.cfg) + var err error + a.key, err = fetchKey(a.cfg) if err != nil { a.log.Fatal(logs.FailedToLoadPrivateKey, zap.Error(err)) } endpoints := fetchMorphEndpoints(a.cfg, a.log) - newListenerFunc := a.getNewListenerFunction(ctx, key, endpoints) - handler := a.getNewEpochHandler() + reconnectInterval := fetchMorphReconnectClientsInterval(a.cfg) + + clientCfg := morph.Config{ + Logger: a.log, + Endpoints: endpoints, + Key: a.key, + ReconnectInterval: reconnectInterval, + DialTimeout: fetchMorphDialTimeout(a.cfg), + } + + cli, err := morph.New(ctx, clientCfg) + if err != nil { + a.log.Fatal(logs.FailedToInitMorphClient, zap.Error(err)) + } + + credSource, err := walletsource.New(fetchWalletsCredentials(a.cfg, a.log)) + if err != nil { + a.log.Fatal(logs.CouldntCreateWalletSource, zap.Error(err)) + } + + frostfsidContract, err := resolver.ResolveContractHash(a.cfg.GetString(cfgMorphContractFrostfsID), endpoints[0].Address) + if err != nil { + a.log.Fatal(logs.ResolveFrostfsIDContract, zap.Error(err)) + } + + ffsidCfg := contract.FrostFSIDConfig{ + Client: cli, + ContractHash: frostfsidContract, + } + + containerContract, err := resolver.ResolveContractHash(a.cfg.GetString(cfgMorphContractContainer), endpoints[0].Address) + if err != nil { + a.log.Fatal(logs.ResolveContainerContract, zap.Error(err)) + } + + containerCfg := contract.ContainerConfig{ + Client: cli, + ContractHash: containerContract, + Log: a.log, + } + + objPool, treePool := getPools(ctx, a.cfg, a.log, a.key) + + epochCh := make(chan uint64) + go func() { + <-a.done + close(epochCh) + }() + + lifecycleCfg := lifecycle.Config{ + UserFetcher: contract.NewFrostFSID(ffsidCfg), + ContainerFetcher: contract.NewContainer(containerCfg), + ConfigurationFetcher: frostfs.NewFrostFS(objPool, a.log), + CredentialSource: credSource, + Settings: a.settings, + CurrentLifecycler: a.key, + Logger: a.log, + TreeFetcher: tree.NewTree(frostfs.NewTreePoolWrapper(treePool), a.log), + BufferSize: fetchJobFetcherBuffer(a.cfg), + EpochChannel: epochCh, + } + + jobProvider := lifecycle.NewJobProvider(ctx, lifecycleCfg) + + go func() { + // todo (d.kirillov) use real job executor here TrueCloudLab/frostfs-s3-lifecycler#4 + for job := range jobProvider.Jobs() { + fmt.Println(job) + } + }() netmapContract, err := resolver.ResolveContractHash(a.cfg.GetString(cfgMorphContractNetmap), endpoints[0].Address) if err != nil { a.log.Fatal(logs.ResolveNetmapContract, zap.Error(err)) } - cfg := notificator.Config{ - Handler: handler, - Logger: a.log, - NewListener: newListenerFunc, - NetmapContract: netmapContract, - ReconnectClientsInterval: 30 * time.Second, + notificatorCfg := notificator.Config{ + Handler: func(ctx context.Context, ee notificator.NewEpochEvent) { + a.log.Info(logs.HandlerTriggered, zap.Uint64("epoch", ee.Epoch)) + select { + case <-ctx.Done(): + a.log.Debug(logs.HandlerContextCanceled, zap.Error(ctx.Err())) + case epochCh <- ee.Epoch: + } + }, + Logger: a.log, + NewListenerFn: func(config notificator.ListenerConfig) (notificator.Listener, error) { + lnCfg := notificator.ConfigListener{ + Client: cli, + Logger: a.log, + ReconnectInterval: reconnectInterval, + Parser: config.Parser, + Handler: config.Handler, + } + + return notificator.NewListener(ctx, lnCfg) + }, + NetmapContract: netmapContract, } - if a.notificator, err = notificator.New(ctx, cfg); err != nil { + if a.notificator, err = notificator.New(ctx, notificatorCfg); err != nil { a.log.Fatal(logs.InitNotificator, zap.Error(err)) } } -func (a *App) getNewListenerFunction(ctx context.Context, key *keys.PrivateKey, endpoints []client.Endpoint) notificator.ListenerCreationFunc { - morphLogger := &logger.Logger{Logger: a.log} - clientOptions := []client.Option{ - client.WithLogger(morphLogger), - client.WithEndpoints(endpoints...), +func newAppSettings(v *viper.Viper, log *Logger) *appSettings { + s := &appSettings{} + + s.update(v, log.logger) + + return s +} + +func (s *appSettings) update(cfg *viper.Viper, log *zap.Logger) { + svcKeys, svcKeyErr := fetchLifecycleServices(cfg) + if svcKeyErr != nil { + log.Warn(logs.FailedToFetchServicesKeys, zap.Error(svcKeyErr)) } - return func(connectionLostCb func()) (event.Listener, error) { - options := append([]client.Option{client.WithConnLostCallback(connectionLostCb)}, clientOptions...) - cli, err := client.New(ctx, key, options...) - if err != nil { - return nil, fmt.Errorf("create new client: %w", err) - } + s.mu.Lock() + defer s.mu.Unlock() - currentBlock, err := cli.BlockCount() - if err != nil { - return nil, fmt.Errorf("get block count: %w", err) - } - - subs, err := subscriber.New(ctx, &subscriber.Params{ - Log: morphLogger, - StartFromBlock: currentBlock, - Client: cli, - }) - if err != nil { - return nil, fmt.Errorf("create subscriber: %w", err) - } - - return event.NewListener(event.ListenerParams{ - Logger: morphLogger, - Subscriber: subs, - WorkerPoolCapacity: 0, // 0 means "infinite" - }) + if svcKeyErr == nil { + s.serviceKeys = svcKeys } } -func (a *App) getNewEpochHandler() notificator.NewEpochHandler { - return func(_ context.Context, ee notificator.NewEpochEvent) { - // todo (d.kirillov) use real job executor here TrueCloudLab/frostfs-s3-lifecycler#3 - fmt.Println("start handler", ee.Epoch) - time.Sleep(30 * time.Second) - fmt.Println("end handler", ee.Epoch) - } +func (s *appSettings) ServicesKeys() keys.PublicKeys { + s.mu.RLock() + defer s.mu.RUnlock() + return s.serviceKeys } func (a *App) Wait() { @@ -187,6 +267,8 @@ func (a *App) configReload() { a.stopAppServices() a.startAppServices() + a.settings.update(a.cfg, a.log) + a.log.Info(logs.SIGHUPConfigReloadCompleted) } @@ -212,3 +294,58 @@ func (a *App) stopAppServices() { svc.ShutDown(ctx) } } + +func getPools(ctx context.Context, cfg *viper.Viper, logger *zap.Logger, key *keys.PrivateKey) (*pool.Pool, *treepool.Pool) { + var prm pool.InitParameters + var prmTree treepool.InitParameters + + prm.SetKey(&key.PrivateKey) + prmTree.SetKey(key) + + for _, peer := range fetchPeers(cfg, logger) { + prm.AddNode(peer) + prmTree.AddNode(peer) + } + + connTimeout := fetchConnectTimeout(cfg) + prm.SetNodeDialTimeout(connTimeout) + prmTree.SetNodeDialTimeout(connTimeout) + + streamTimeout := fetchStreamTimeout(cfg) + prm.SetNodeStreamTimeout(streamTimeout) + prmTree.SetNodeStreamTimeout(streamTimeout) + + healthCheckTimeout := fetchHealthCheckTimeout(cfg) + prm.SetHealthcheckTimeout(healthCheckTimeout) + prmTree.SetHealthcheckTimeout(healthCheckTimeout) + + rebalanceInterval := fetchRebalanceInterval(cfg) + prm.SetClientRebalanceInterval(rebalanceInterval) + prmTree.SetClientRebalanceInterval(rebalanceInterval) + + errorThreshold := fetchErrorThreshold(cfg) + prm.SetErrorThreshold(errorThreshold) + prm.SetLogger(logger) + prmTree.SetLogger(logger) + + prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgFrostFSTreePoolMaxAttempts)) + + p, err := pool.NewPool(prm) + if err != nil { + logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) + } + + if err = p.Dial(ctx); err != nil { + logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + } + + treePool, err := treepool.NewPool(prmTree) + if err != nil { + logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) + } + if err = treePool.Dial(ctx); err != nil { + logger.Fatal(logs.FailedToDialTreePool, zap.Error(err)) + } + + return p, treePool +} diff --git a/cmd/s3-lifecycler/settings.go b/cmd/s3-lifecycler/settings.go index 17902e8..4543259 100644 --- a/cmd/s3-lifecycler/settings.go +++ b/cmd/s3-lifecycler/settings.go @@ -9,8 +9,10 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/credential/walletsource" "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs" - "github.com/nspcc-dev/neo-go/cli/flags" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + neogoflags "github.com/nspcc-dev/neo-go/cli/flags" "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" @@ -43,8 +45,33 @@ const ( cfgMorphRPCEndpointTrustedCAListTmpl = cfgMorphRPCEndpointPrefixTmpl + "trusted_ca_list" cfgMorphRPCEndpointCertificateTmpl = cfgMorphRPCEndpointPrefixTmpl + "certificate" cfgMorphRPCEndpointKeyTmpl = cfgMorphRPCEndpointPrefixTmpl + "key" + cfgMorphReconnectClientsInterval = "morph.reconnect_clients_interval" + cfgMorphDialTimeout = "morph.dial_timeout" cfgMorphContractNetmap = "morph.contract.netmap" - cfgMorphReconnectClientInterval = "morph.reconnect_clients_interval" + cfgMorphContractFrostfsID = "morph.contract.frostfsid" + cfgMorphContractContainer = "morph.contract.container" + + // Credential source. + cfgCredentialSourceWalletsPrefixTmpl = "credential_source.wallets.%d." + cfgCredentialSourceWalletsPathTmpl = cfgCredentialSourceWalletsPrefixTmpl + "path" + cfgCredentialSourceWalletsAddressTmpl = cfgCredentialSourceWalletsPrefixTmpl + "address" + cfgCredentialSourceWalletsPassphraseTmpl = cfgCredentialSourceWalletsPrefixTmpl + "passphrase" + + // FrostFS. + cfgFrostFSConnectTimeout = "frostfs.connect_timeout" + cfgFrostFSStreamTimeout = "frostfs.stream_timeout" + cfgFrostFSHealthcheckTimeout = "frostfs.healthcheck_timeout" + cfgFrostFSRebalanceInterval = "frostfs.rebalance_interval" + cfgFrostFSPoolErrorThreshold = "frostfs.pool_error_threshold" + cfgFrostFSTreePoolMaxAttempts = "frostfs.tree_pool_max_attempts" + cfgFrostFSPeersPrefixTmpl = "frostfs.peers.%d." + cfgFrostFSPeersAddressTmpl = cfgFrostFSPeersPrefixTmpl + "address" + cfgFrostFSPeersPriorityTmpl = cfgFrostFSPeersPrefixTmpl + "priority" + cfgFrostFSPeersWeightTmpl = cfgFrostFSPeersPrefixTmpl + "weight" + + // Lifecycle. + cfgLifecycleJobFetcherBuffer = "lifecycle.job_fetcher_buffer" + cfgLifecycleServices = "lifecycle.services" // Command line args. cmdHelp = "help" @@ -57,7 +84,17 @@ const ( defaultShutdownTimeout = 15 * time.Second componentName = "frostfs-s3-lifecycler" - defaultMorphRPCEndpointPriority = 1 + defaultMorphRPCEndpointPriority = 1 + defaultMorphReconnectClientsInterval = 30 * time.Second + defaultMorphDialTimeout = 5 * time.Second + + defaultFrostFSRebalanceInterval = 60 * time.Second + defaultFrostFSHealthcheckTimeout = 15 * time.Second + defaultFrostFSConnectTimeout = 10 * time.Second + defaultFrostFSStreamTimeout = 10 * time.Second + defaultFrostFSPoolErrorThreshold uint32 = 100 + + defaultLifecycleJobFetcherBuffer = 1000 ) func settings() *viper.Viper { @@ -90,8 +127,20 @@ func settings() *viper.Viper { v.SetDefault(cfgPprofEnabled, false) // morph: + v.SetDefault(cfgMorphReconnectClientsInterval, defaultMorphReconnectClientsInterval) + v.SetDefault(cfgMorphDialTimeout, defaultMorphDialTimeout) v.SetDefault(cfgMorphContractNetmap, "netmap.frostfs") - v.SetDefault(cfgMorphReconnectClientInterval, 30*time.Second) + v.SetDefault(cfgMorphContractFrostfsID, "frostfsid.frostfs") + v.SetDefault(cfgMorphContractContainer, "container.frostfs") + + // frostfs: + v.SetDefault(cfgFrostFSConnectTimeout, defaultFrostFSConnectTimeout) + v.SetDefault(cfgFrostFSRebalanceInterval, defaultFrostFSRebalanceInterval) + v.SetDefault(cfgFrostFSHealthcheckTimeout, defaultFrostFSHealthcheckTimeout) + v.SetDefault(cfgFrostFSStreamTimeout, defaultFrostFSStreamTimeout) + + // lifecycle: + v.SetDefault(cfgLifecycleJobFetcherBuffer, defaultLifecycleJobFetcherBuffer) // Bind flags with configuration values. if err := v.BindPFlags(flags); err != nil { @@ -212,7 +261,7 @@ func fetchKey(v *viper.Viper) (*keys.PrivateKey, error) { if len(walletAddress) == 0 { addr = w.GetChangeAddress() } else { - addr, err = flags.ParseAddress(walletAddress) + addr, err = neogoflags.ParseAddress(walletAddress) if err != nil { return nil, fmt.Errorf("invalid address") } @@ -273,3 +322,146 @@ func fetchMorphEndpoints(v *viper.Viper, l *zap.Logger) []client.Endpoint { } return res } + +func fetchWalletsCredentials(v *viper.Viper, l *zap.Logger) []walletsource.Wallet { + var res []walletsource.Wallet + + for i := 0; ; i++ { + walletPath := v.GetString(fmt.Sprintf(cfgCredentialSourceWalletsPathTmpl, i)) + if walletPath == "" { + break + } + + res = append(res, walletsource.Wallet{ + Path: walletPath, + Address: v.GetString(fmt.Sprintf(cfgCredentialSourceWalletsAddressTmpl, i)), + Passphrase: v.GetString(fmt.Sprintf(cfgCredentialSourceWalletsPassphraseTmpl, i)), + }) + } + + if len(res) == 0 { + l.Fatal(logs.NoCredentialSourceWallets) + } + return res +} + +func fetchPeers(v *viper.Viper, l *zap.Logger) []pool.NodeParam { + var nodes []pool.NodeParam + for i := 0; ; i++ { + address := v.GetString(fmt.Sprintf(cfgFrostFSPeersAddressTmpl, i)) + if address == "" { + break + } + + priority := v.GetInt(fmt.Sprintf(cfgFrostFSPeersPriorityTmpl, i)) + if priority <= 0 { // unspecified or wrong + priority = 1 + } + + weight := v.GetFloat64(fmt.Sprintf(cfgFrostFSPeersWeightTmpl, i)) + if weight <= 0 { // unspecified or wrong + weight = 1 + } + + nodes = append(nodes, pool.NewNodeParam(priority, address, weight)) + + l.Info(logs.AddedStoragePeer, + zap.String("address", address), + zap.Int("priority", priority), + zap.Float64("weight", weight)) + } + + return nodes +} + +func fetchConnectTimeout(cfg *viper.Viper) time.Duration { + connTimeout := cfg.GetDuration(cfgFrostFSConnectTimeout) + if connTimeout <= 0 { + connTimeout = defaultFrostFSConnectTimeout + } + + return connTimeout +} + +func fetchStreamTimeout(cfg *viper.Viper) time.Duration { + streamTimeout := cfg.GetDuration(cfgFrostFSStreamTimeout) + if streamTimeout <= 0 { + streamTimeout = defaultFrostFSStreamTimeout + } + + return streamTimeout +} + +func fetchHealthCheckTimeout(cfg *viper.Viper) time.Duration { + healthCheckTimeout := cfg.GetDuration(cfgFrostFSHealthcheckTimeout) + if healthCheckTimeout <= 0 { + healthCheckTimeout = defaultFrostFSHealthcheckTimeout + } + + return healthCheckTimeout +} + +func fetchRebalanceInterval(cfg *viper.Viper) time.Duration { + rebalanceInterval := cfg.GetDuration(cfgFrostFSRebalanceInterval) + if rebalanceInterval <= 0 { + rebalanceInterval = defaultFrostFSRebalanceInterval + } + + return rebalanceInterval +} + +func fetchErrorThreshold(cfg *viper.Viper) uint32 { + errorThreshold := cfg.GetUint32(cfgFrostFSPoolErrorThreshold) + if errorThreshold <= 0 { + errorThreshold = defaultFrostFSPoolErrorThreshold + } + + return errorThreshold +} + +func fetchJobFetcherBuffer(cfg *viper.Viper) int { + bufferSize := cfg.GetInt(cfgLifecycleJobFetcherBuffer) + if bufferSize <= 0 { + bufferSize = defaultLifecycleJobFetcherBuffer + } + + return bufferSize +} + +func fetchMorphReconnectClientsInterval(cfg *viper.Viper) time.Duration { + val := cfg.GetDuration(cfgMorphReconnectClientsInterval) + if val <= 0 { + val = defaultMorphReconnectClientsInterval + } + + return val +} + +func fetchMorphDialTimeout(cfg *viper.Viper) time.Duration { + val := cfg.GetDuration(cfgMorphDialTimeout) + if val <= 0 { + val = defaultMorphDialTimeout + } + + return val +} + +func fetchLifecycleServices(v *viper.Viper) (keys.PublicKeys, error) { + configKeys := v.GetStringSlice(cfgLifecycleServices) + result := make(keys.PublicKeys, 0, len(configKeys)) + uniqKeys := make(map[string]struct{}, len(configKeys)) + + for _, configKey := range configKeys { + if _, ok := uniqKeys[configKey]; ok { + continue + } + k, err := keys.NewPublicKeyFromString(configKey) + if err != nil { + return nil, fmt.Errorf("key '%s': %w", configKey, err) + } + result = append(result, k) + uniqKeys[configKey] = struct{}{} + } + + return result, nil +} diff --git a/config/config.env b/config/config.env index bb4baf2..a3e511b 100644 --- a/config/config.env +++ b/config/config.env @@ -26,4 +26,34 @@ S3_LIFECYCLER_MORPH_RPC_ENDPOINT_0_KEY="/path/to/key" S3_LIFECYCLER_MORPH_RPC_ENDPOINT_1_ADDRESS="wss://rpc2.morph.frostfs.info:40341/ws" S3_LIFECYCLER_MORPH_RPC_ENDPOINT_1_PRIORITY=2 S3_LIFECYCLER_MORPH_RECONNECT_CLIENTS_INTERVAL=30s +S3_LIFECYCLER_MORPH_RECONNECT_DIAL_TIMEOUT=5s S3_LIFECYCLER_MORPH_CONTRACT_NETMAP=netmap.frostfs +S3_LIFECYCLER_MORPH_CONTRACT_FROSTFSID=frostfsid.frostfs +S3_LIFECYCLER_MORPH_CONTRACT_CONTAINER=container.frostfs + +# Credential source +S3_LIFECYCLER_CREDENTIAL_SOURCE_WALLETS_0_PATH=/path/to/user/wallet.json +S3_LIFECYCLER_CREDENTIAL_SOURCE_WALLETS_0_ADDRESS=NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP +S3_LIFECYCLER_CREDENTIAL_SOURCE_WALLETS_0_PASSPHRASE="" + +# Lifecycle +S3_LIFECYCLER_LIFECYCLE_JOB_FETCHER_BUFFER=1000 +S3_LIFECYCLER_LIFECYCLE_SERVICES=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a + +# FrostFS +S3_LIFECYCLER_FROSTFS_STREAM_TIMEOUT=10s +S3_LIFECYCLER_FROSTFS_CONNECT_TIMEOUT=10s +S3_LIFECYCLER_FROSTFS_HEALTHCHECK_TIMEOUT=15s +S3_LIFECYCLER_FROSTFS_REBALANCE_INTERVAL=60s +S3_LIFECYCLER_FROSTFS_POOL_ERROR_THRESHOLD=100 +S3_LIFECYCLER_FROSTFS_TREE_POOL_MAX_ATTEMPTS=4 + +S3_LIFECYCLER_FROSTFS_PEERS_0_ADDRESS=node1.frostfs:8080 +S3_LIFECYCLER_FROSTFS_PEERS_0_PRIORITY=1 +S3_LIFECYCLER_FROSTFS_PEERS_0_WEIGHT=1 +S3_LIFECYCLER_FROSTFS_PEERS_1_ADDRESS=node2.frostfs:8080 +S3_LIFECYCLER_FROSTFS_PEERS_1_PRIORITY=2 +S3_LIFECYCLER_FROSTFS_PEERS_1_WEIGHT=0.1 +S3_LIFECYCLER_FROSTFS_PEERS_2_ADDRESS=node3.frostfs:8080 +S3_LIFECYCLER_FROSTFS_PEERS_2_PRIORITY=2 +S3_LIFECYCLER_FROSTFS_PEERS_2_WEIGHT=0.9 diff --git a/config/config.yaml b/config/config.yaml index 28a4ad7..af60c2e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -27,5 +27,45 @@ morph: - address: wss://rpc2.morph.frostfs.info:40341/ws priority: 2 reconnect_clients_interval: 30s + dial_timeout: 5s contract: netmap: netmap.frostfs + frostfsid: frostfsid.frostfs + container: container.frostfs + +credential_source: + wallets: + - path: /path/to/wallet.json + address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP + passphrase: "" + +lifecycle: + job_fetcher_buffer: 1000 + services: + - 0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf + +frostfs: + stream_timeout: 10s + connect_timeout: 10s + healthcheck_timeout: 15s + rebalance_interval: 60s + pool_error_threshold: 100 + tree_pool_max_attempts: 4 + + peers: + 0: + priority: 1 + weight: 1 + address: s01.frostfs.devenv:8080 + 1: + priority: 2 + weight: 1 + address: s02.frostfs.devenv:8080 + 2: + priority: 2 + weight: 1 + address: s03.frostfs.devenv:8080 + 3: + priority: 2 + weight: 1 + address: s04.frostfs.devenv:8080 diff --git a/docs/configuration.md b/docs/configuration.md index 0189b2a..6f6b094 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -4,12 +4,16 @@ This section contains detailed FrostFS S3 Lifecycler component configuration des # Structure -| Section | Description | -|--------------|-------------------------------------------------| -| `logger` | [Logger configuration](#logger-section) | -| `pprof` | [Pprof configuration](#pprof-section) | -| `prometheus` | [Prometheus configuration](#prometheus-section) | -| `morph` | [Morph configuration](#morph-section) | +| Section | Description | +|---------------------|--------------------------------------------------------------| +| `wallet` | [Wallet configuration](#wallet-section) | +| `logger` | [Logger configuration](#logger-section) | +| `pprof` | [Pprof configuration](#pprof-section) | +| `prometheus` | [Prometheus configuration](#prometheus-section) | +| `morph` | [Morph configuration](#morph-section) | +| `credential_source` | [Credential source configuration](#credentialsource-section) | +| `lifecycle` | [Lifecycle configuration](#lifecycle-section) | +| `frostfs` | [FrostFS configuration](#frostfs-section) | ### Reload on SIGHUP @@ -22,6 +26,23 @@ You can send SIGHUP signal to app using the following command: $ kill -s SIGHUP ``` +# `wallet` section + +Configuration of key for lifecycle service. + +```yaml +wallet: + path: /path/to/wallet.json + address: Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn + passphrase: "" +``` + +| Parameter | Type | Default value | Description | +|--------------|----------|---------------|--------------------------------------------------------------------------| +| `path` | `string` | | Path to wallet | +| `address` | `string` | | Account address to get from wallet. If omitted default one will be used. | +| `passphrase` | `string` | | Passphrase to decrypt wallet. | + # `logger` section ```yaml @@ -81,16 +102,108 @@ morph: - address: wss://rpc2.morph.frostfs.info:40341/ws priority: 2 reconnect_clients_interval: 30s + dial_timeout: 5s contract: netmap: netmap.frostfs + frostfsid: frostfsid.frostfs ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|--------------------------------|------------|---------------|------------------|---------------------------------------------------------------------------------------------------------| -| `rpc_endpoint.address` | `string` | no | | The address of the RPC host to connect. | -| `rpc_endpoint.priority` | `int` | no | | Priority of RPC endpoint. | -| `rpc_endpoint.trusted_ca_list` | `[]string` | no | | List of paths to CAs to use in mTLS configuration. | -| `rpc_endpoint.certificate` | `string` | no | | Path to certificate to use in mTLS configuration. | -| `rpc_endpoint.key` | `string` | no | | Path to key to use in mTLS configuration. | -| `reconnect_clients_interval` | `string` | no | `30s` | When all endpoints are failed. Overall connection be reinitialized. This value is time between retries. | -| `contract.netmap` | `string` | no | `netmap.frostfs` | Netmap contract hash (LE) or name in NNS. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|--------------------------------|------------|---------------|---------------------|------------------------------------------------------------------------------------------------------------------| +| `rpc_endpoint.address` | `string` | no | | The address of the RPC host to connect. | +| `rpc_endpoint.priority` | `int` | no | | Priority of RPC endpoint. | +| `rpc_endpoint.trusted_ca_list` | `[]string` | no | | List of paths to CAs to use in mTLS configuration. | +| `rpc_endpoint.certificate` | `string` | no | | Path to certificate to use in mTLS configuration. | +| `rpc_endpoint.key` | `string` | no | | Path to key to use in mTLS configuration. | +| `reconnect_clients_interval` | `string` | no | `30s` | When all endpoints are failed. Overall connection be reinitialized. This value is time between retries. | +| `reconnect_clients_interval` | `string` | no | `5s` | Dial timeout to connect to morph endpoint. | +| `contract.netmap` | `string` | no | `netmap.frostfs` | Netmap contract hash (LE) or name in NNS. | +| `contract.frostfsid` | `string` | no | `frostfsid.frostfs` | FrostfsID contract hash (LE) or name in NNS. This contract is used to get all users to process their containers. | +| `contract.container` | `string` | no | `container.frostfs` | Container contract hash (LE) or name in NNS. | + +# `credential_source` section + +Contains configuration for the source of user private keys (credentials). + +```yaml +credential_source: + wallets: + - path: /path/to/wallet.json + address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP + passphrase: "" +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|----------------------|----------|---------------|---------------|-----------------------------------------------------------------| +| `wallets` | | | | Source of user private keys as wallets files on filesystem. | +| `wallets.path` | `string` | no | | Path to wallet on filesystem. | +| `wallets.address` | `string` | no | | Account address in wallet. If omitted default one will be used. | +| `wallets.passphrase` | `string` | no | | Passphrase to decrypt wallet. | + +# `lifecycle` section + +Configuration for main lifecycle handling procedure. + +```yaml +lifecycle: + job_fetcher_buffer: 1000 + services: + - 0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|----------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `job_fetcher_buffer` | `int` | no | `1000` | Size for buffered channel to fetch users/container and other data for lifecycle procedure. This param helps reduce number concurrent outgoing network requests. | +| `services` | `[]string` | yes | | List of Lifecycle services public keys. Needs to split jobs. | + +# `frostfs` section + +Configuration for FrostFS storage. + +```yaml +frostfs: + stream_timeout: 10s + connect_timeout: 10s + healthcheck_timeout: 5s + rebalance_interval: 1m + pool_error_threshold: 100 + tree_pool_max_attempts: 4 + + peers: + 0: + address: node1.frostfs:8080 + priority: 1 + weight: 1 + 1: + address: node2.frostfs:8080 + priority: 2 + weight: 0.1 + 2: + address: node3.frostfs:8080 + priority: 2 + weight: 0.9 +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|--------------------------|------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------| +| `stream_timeout` | `duration` | no | `10s` | Timeout for individual operations in streaming RPC. | +| `connect_timeout` | `duration` | no | `10s` | Timeout to connect to a storage node. | +| `healthcheck_timeout` | `duration` | no | `15s` | Timeout to check storage node health during rebalance. | +| `rebalance_interval` | `duration` | no | `60s` | Interval to check storage node health. | +| `pool_error_threshold` | `uint32` | no | `100` | The number of errors on connection after which storage node is considered as unhealthy. | +| `tree_pool_max_attempts` | `uint32` | no | `0` | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. | + +## `peers` section + +This configuration makes TO-IAM use the first node (node1.frostfs:8080) +while it's healthy. Otherwise, TO-IAM uses the second node (node2.frostfs:8080) +for 10% of requests and the third node (node3.frostfs:8080) for 90% of requests. +Until nodes with the same priority level are healthy +nodes with other priority are not used. +The lower the value, the higher the priority. + +| Parameter | Type | Default value | Description | +|------------------|----------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| +| `peers.address` | `string` | | Address of storage node. | +| `peers.priority` | `int` | `1` | It allows to group nodes and don't switch group until all nodes with the same priority will be unhealthy. The lower the value, the higher the priority. | +| `peers.weight` | `float` | `1` | Weight of node in the group with the same priority. Distribute requests to nodes proportionally to these values. | diff --git a/go.mod b/go.mod index 6ebe228..2c126a2 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,12 @@ module git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler go 1.21 require ( + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-node v0.42.0-rc.5 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de + git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.30.0-rc.4.0.20240709102501-0e1ab11a1bd7 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4 + git.frostfs.info/TrueCloudLab/hrw v1.2.1 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/nspcc-dev/neo-go v0.106.0 github.com/prometheus/client_golang v1.19.1 @@ -14,30 +18,41 @@ require ( github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.27.0 + golang.org/x/text v0.16.0 ) replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 // indirect - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 // indirect git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 // indirect - git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect + git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/aws/aws-sdk-go v1.44.6 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bluele/gcache v0.0.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-chi/chi/v5 v5.0.8 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/minio/sio v0.3.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect @@ -59,13 +74,22 @@ require ( github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.14 // indirect go.etcd.io/bbolt v1.3.9 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/sdk v1.22.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/net v0.23.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3ae3de6 --- /dev/null +++ b/go.sum @@ -0,0 +1,331 @@ +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= +git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= +git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= +git.frostfs.info/TrueCloudLab/frostfs-node v0.42.0-rc.5 h1:lVWO3JtF3R4Irb+/xT5+wY0oMOPgRTytHichxm+nIjk= +git.frostfs.info/TrueCloudLab/frostfs-node v0.42.0-rc.5/go.mod h1:IZBD+sRxSxpXXIkg0rAK5yvkGHZUaHBqmcWFu2UmbmQ= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= +git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.30.0-rc.4.0.20240709102501-0e1ab11a1bd7 h1:MENM+TjT05VtHQoxu0AjeyTVWH48/piizYpHpATFhNI= +git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.30.0-rc.4.0.20240709102501-0e1ab11a1bd7/go.mod h1:y5tKPMT0xgKwb6TEMWzBgdmRqFWTWXckX03u55/4Gyg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4 h1:izmHYpkz7cPr2Zpudxxh0wvrtAIxYywEG+uraghVSlo= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= +git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= +git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 h1:JRjwcHaQajTbSCBCK3yZnqvyHvgWBaoThDGuT4kvIIc= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a h1:Bk1fB4cQASPKgAVGCdlBOEp5ohZfDxqK6fZM8eP+Emo= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= +git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= +git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= +git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= +git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= +git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 h1:HeY8n27VyPRQe49l/fzyVMkWEB2fsLJYKp64pwA7tz4= +git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/aws/aws-sdk-go v1.44.6 h1:Y+uHxmZfhRTLX2X3khkdxCoTZAyGEX21aOUHe1U6geg= +github.com/aws/aws-sdk-go v1.44.6/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 h1:tYj5Ydh5D7Xg2R1tJnoG36Yta7NVB8C0vx36oPA3Bbw= +github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2/go.mod h1:wKqwsieaKPThcFkHe0d0zMsbHEUWFmZcG7KBCse210o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= +github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= +github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= +github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= +github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d h1:Vcb7YkZuUSSIC+WF/xV3UDfHbAxZgyT2zGleJP3Ig5k= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY= +github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/RgRM= +github.com/nspcc-dev/rfc6979 v0.2.1/go.mod h1:Tk7h5kyUWkhjyO3zUgFFhy1v2vQv3BvQEntakdtqrWc= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/panjf2000/ants/v2 v2.9.0 h1:SztCLkVxBRigbg+vt0S5QvF5vxAbxbKt09/YfAJ0tEo= +github.com/panjf2000/ants/v2 v2.9.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/ssgreg/journald v1.0.0 h1:0YmTDPJXxcWDPba12qNMdO6TxvfkFSYpFIJ31CwmLcU= +github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4JEP+oRt0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= +github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= +go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 h1:H2JFgRcGiyHg7H7bwcwaQJYrNFqCqrbTQ8K4p1OvDu8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0/go.mod h1:WfCWp1bGoYK8MeULtI15MmQVczfR+bFkk0DF3h06QmQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 h1:zr8ymM5OWWjjiWRzwTfZ67c905+2TMHYp2lMJ52QTyM= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0/go.mod h1:sQs7FT2iLVJ+67vYngGJkPe1qr39IzaBzaj9IDNNY8k= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= +go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/internal/credential/walletsource/wallet.go b/internal/credential/walletsource/wallet.go new file mode 100644 index 0000000..4fb6157 --- /dev/null +++ b/internal/credential/walletsource/wallet.go @@ -0,0 +1,77 @@ +package walletsource + +import ( + "context" + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/lifecycle" + "github.com/nspcc-dev/neo-go/cli/flags" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/wallet" +) + +type Source struct { + keys []*keys.PrivateKey +} + +type Wallet struct { + Path string + Address string + Passphrase string +} + +var _ lifecycle.CredentialSource = (*Source)(nil) + +func New(wallets []Wallet) (*Source, error) { + privateKeys := make([]*keys.PrivateKey, len(wallets)) + + var err error + for i, w := range wallets { + if privateKeys[i], err = readPrivateKey(w); err != nil { + return nil, fmt.Errorf("read private key from wallet '%s': %w", w.Path, err) + } + } + + return &Source{keys: privateKeys}, nil +} + +func (s *Source) Credentials(_ context.Context, pk *keys.PublicKey) (*keys.PrivateKey, error) { + for _, key := range s.keys { + if key.PublicKey().Equal(pk) { + return key, nil + } + } + + return nil, errors.New("key not found") +} + +func readPrivateKey(walletInfo Wallet) (*keys.PrivateKey, error) { + w, err := wallet.NewWalletFromFile(walletInfo.Path) + if err != nil { + return nil, fmt.Errorf("parse wallet: %w", err) + } + + var addr util.Uint160 + if walletInfo.Address == "" { + addr = w.GetChangeAddress() + } else { + addr, err = flags.ParseAddress(walletInfo.Address) + if err != nil { + return nil, fmt.Errorf("invalid address") + } + } + + acc := w.GetAccount(addr) + if acc == nil { + return nil, fmt.Errorf("couldn't find wallet account for %s", address.Uint160ToString(addr)) + } + + if err = acc.Decrypt(walletInfo.Passphrase, w.Scrypt); err != nil { + return nil, fmt.Errorf("couldn't decrypt account: %w", err) + } + + return acc.PrivateKey(), nil +} diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go new file mode 100644 index 0000000..c1c28d8 --- /dev/null +++ b/internal/frostfs/frostfs.go @@ -0,0 +1,101 @@ +package frostfs + +import ( + "context" + "encoding/xml" + "fmt" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "go.uber.org/zap" + "golang.org/x/text/encoding/ianaindex" +) + +// FrostFS represents virtual connection to the FrostFS network. +// It is used to provide an interface to dependent packages +// which work with FrostFS. +type FrostFS struct { + pool *pool.Pool + log *zap.Logger +} + +// NewFrostFS creates new FrostFS using provided pool.Pool. +func NewFrostFS(p *pool.Pool, log *zap.Logger) *FrostFS { + return &FrostFS{ + pool: p, + log: log, + } +} + +type PrmGetObject struct { + // Container to read the object header from. + Container cid.ID + + // ID of the object for which to read the header. + Object oid.ID + + // Bearer token to be used for the operation. Overlaps PrivateKey. Optional. + BearerToken bearer.Token +} + +func (f *FrostFS) GetObject(ctx context.Context, prm PrmGetObject) (pool.ResGetObject, error) { + var addr oid.Address + addr.SetContainer(prm.Container) + addr.SetObject(prm.Object) + + var prmGet pool.PrmObjectGet + prmGet.SetAddress(addr) + prmGet.UseBearer(prm.BearerToken) + + return f.pool.GetObject(ctx, prmGet) +} + +func (f *FrostFS) LifecycleConfiguration(ctx context.Context, addr oid.Address) (*data.LifecycleConfiguration, error) { + prm := PrmGetObject{ + Container: addr.Container(), + Object: addr.Object(), + } + + if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate.BearerToken != nil { + prm.BearerToken = *bd.Gate.BearerToken + } + + res, err := f.GetObject(ctx, prm) + if err != nil { + return nil, err + } + defer func() { + if closeErr := res.Payload.Close(); closeErr != nil { + f.log.Warn("could not close object payload", zap.String("address", addr.EncodeToString()), zap.Error(closeErr)) + } + }() + + lifecycleCfg := &data.LifecycleConfiguration{} + dec := newDecoder(res.Payload) + if err = dec.Decode(lifecycleCfg); err != nil { + return nil, fmt.Errorf("unmarshal lifecycle configuration '%s': %w", addr.EncodeToString(), err) + } + + return lifecycleCfg, nil +} + +const awsDefaultNamespace = "http://s3.amazonaws.com/doc/2006-03-01/" + +func newDecoder(r io.Reader) *xml.Decoder { + dec := xml.NewDecoder(r) + dec.DefaultSpace = awsDefaultNamespace + dec.CharsetReader = func(charset string, reader io.Reader) (io.Reader, error) { + enc, err := ianaindex.IANA.Encoding(charset) + if err != nil { + return nil, fmt.Errorf("charset %s: %w", charset, err) + } + return enc.NewDecoder().Reader(reader), nil + } + + return dec +} diff --git a/internal/frostfs/tree.go b/internal/frostfs/tree.go new file mode 100644 index 0000000..46e0cb8 --- /dev/null +++ b/internal/frostfs/tree.go @@ -0,0 +1,252 @@ +package frostfs + +import ( + "context" + "errors" + "fmt" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" + grpcService "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree/service" +) + +type GetNodeByPathResponseInfoWrapper struct { + response *grpcService.GetNodeByPathResponse_Info +} + +func (n GetNodeByPathResponseInfoWrapper) GetNodeID() uint64 { + return n.response.GetNodeId() +} + +func (n GetNodeByPathResponseInfoWrapper) GetParentID() uint64 { + return n.response.GetParentId() +} + +func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() uint64 { + return n.response.GetTimestamp() +} + +func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} + +type GetSubTreeResponseBodyWrapper struct { + response *grpcService.GetSubTreeResponse_Body +} + +func (n GetSubTreeResponseBodyWrapper) GetNodeID() uint64 { + return n.response.GetNodeId() +} + +func (n GetSubTreeResponseBodyWrapper) GetParentID() uint64 { + return n.response.GetParentId() +} + +func (n GetSubTreeResponseBodyWrapper) GetTimestamp() uint64 { + return n.response.GetTimestamp() +} + +func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} + +type TreePoolWrapper struct { + p *treepool.Pool +} + +func NewTreePoolWrapper(p *treepool.Pool) *TreePoolWrapper { + return &TreePoolWrapper{p: p} +} + +func (w *TreePoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]tree.NodeResponse, error) { + poolPrm := treepool.GetNodesParams{ + CID: prm.BktInfo.CID, + TreeID: prm.TreeID, + Path: prm.Path, + Meta: prm.Meta, + PathAttribute: tree.FileNameKey, + LatestOnly: prm.LatestOnly, + AllAttrs: prm.AllAttrs, + BearerToken: getBearer(ctx, prm.BktInfo), + } + + nodes, err := w.p.GetNodes(ctx, poolPrm) + if err != nil { + return nil, handleError(err) + } + + res := make([]tree.NodeResponse, len(nodes)) + for i, info := range nodes { + res[i] = GetNodeByPathResponseInfoWrapper{info} + } + + return res, nil +} + +func (w *TreePoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]tree.NodeResponse, error) { + poolPrm := treepool.GetSubTreeParams{ + CID: bktInfo.CID, + TreeID: treeID, + RootID: rootID, + Depth: depth, + BearerToken: getBearer(ctx, bktInfo), + } + + subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) + if err != nil { + return nil, handleError(err) + } + + var subtree []tree.NodeResponse + + node, err := subTreeReader.Next() + for err == nil { + subtree = append(subtree, GetSubTreeResponseBodyWrapper{node}) + node, err = subTreeReader.Next() + } + if err != nil && err != io.EOF { + return nil, handleError(err) + } + + return subtree, nil +} + +type SubTreeStreamImpl struct { + r *treepool.SubTreeReader + buffer []*grpcService.GetSubTreeResponse_Body + eof bool + index int + ln int +} + +const bufSize = 1000 + +func (s *SubTreeStreamImpl) Next() (tree.NodeResponse, error) { + if s.index != -1 { + node := s.buffer[s.index] + s.index++ + if s.index >= s.ln { + s.index = -1 + } + return GetSubTreeResponseBodyWrapper{response: node}, nil + } + if s.eof { + return nil, io.EOF + } + + var err error + s.ln, err = s.r.Read(s.buffer) + if err != nil { + if err != io.EOF { + return nil, fmt.Errorf("sub tree stream impl pool wrap: %w", handleError(err)) + } + s.eof = true + } + if s.ln > 0 { + s.index = 0 + } + + return s.Next() +} + +func (w *TreePoolWrapper) GetSubTreeStream(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) (tree.SubTreeStream, error) { + poolPrm := treepool.GetSubTreeParams{ + CID: bktInfo.CID, + TreeID: treeID, + RootID: rootID, + Depth: depth, + BearerToken: getBearer(ctx, bktInfo), + Order: treepool.AscendingOrder, + } + + subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) + if err != nil { + return nil, handleError(err) + } + + return &SubTreeStreamImpl{ + r: subTreeReader, + buffer: make([]*grpcService.GetSubTreeResponse_Body, bufSize), + index: -1, + }, nil +} + +func (w *TreePoolWrapper) AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) { + nodeID, err := w.p.AddNode(ctx, treepool.AddNodeParams{ + CID: bktInfo.CID, + TreeID: treeID, + Parent: parent, + Meta: meta, + BearerToken: getBearer(ctx, bktInfo), + }) + return nodeID, handleError(err) +} + +func (w *TreePoolWrapper) AddNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) { + nodeID, err := w.p.AddNodeByPath(ctx, treepool.AddNodeByPathParams{ + CID: bktInfo.CID, + TreeID: treeID, + Path: path, + Meta: meta, + PathAttribute: tree.FileNameKey, + BearerToken: getBearer(ctx, bktInfo), + }) + return nodeID, handleError(err) +} + +func (w *TreePoolWrapper) MoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error { + return handleError(w.p.MoveNode(ctx, treepool.MoveNodeParams{ + CID: bktInfo.CID, + TreeID: treeID, + NodeID: nodeID, + ParentID: parentID, + Meta: meta, + BearerToken: getBearer(ctx, bktInfo), + })) +} + +func (w *TreePoolWrapper) RemoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error { + return handleError(w.p.RemoveNode(ctx, treepool.RemoveNodeParams{ + CID: bktInfo.CID, + TreeID: treeID, + NodeID: nodeID, + BearerToken: getBearer(ctx, bktInfo), + })) +} + +func getBearer(ctx context.Context, bktInfo *data.BucketInfo) []byte { + if bd, err := middleware.GetBoxData(ctx); err == nil { + if bd.Gate.BearerToken != nil { + if bd.Gate.BearerToken.Impersonate() || bktInfo.Owner.Equals(bearer.ResolveIssuer(*bd.Gate.BearerToken)) { + return bd.Gate.BearerToken.Marshal() + } + } + } + return nil +} + +func handleError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, treepool.ErrNodeNotFound) { + return fmt.Errorf("%w: %s", tree.ErrNodeNotFound, err.Error()) + } + if errors.Is(err, treepool.ErrNodeAccessDenied) { + return fmt.Errorf("%w: %s", tree.ErrNodeAccessDenied, err.Error()) + } + + return err +} diff --git a/internal/lifecycle/fetcher.go b/internal/lifecycle/fetcher.go new file mode 100644 index 0000000..2d30072 --- /dev/null +++ b/internal/lifecycle/fetcher.go @@ -0,0 +1,392 @@ +package lifecycle + +import ( + "context" + "crypto/ecdsa" + "encoding/binary" + "encoding/hex" + "fmt" + "slices" + "sort" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/hrw" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/util" + "go.uber.org/zap" +) + +type UserFetcher interface { + Users() ([]util.Uint160, error) + UserKey(hash util.Uint160) (*keys.PublicKey, error) +} + +type ContainerFetcher interface { + Containers(owner user.ID) ([]cid.ID, error) +} + +type TreeFetcher interface { + GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) +} + +type ConfigurationFetcher interface { + LifecycleConfiguration(ctx context.Context, addr oid.Address) (*data.LifecycleConfiguration, error) +} + +type CredentialSource interface { + Credentials(ctx context.Context, pk *keys.PublicKey) (*keys.PrivateKey, error) +} + +type Job struct { + ContainerID cid.ID + PrivateKey *keys.PrivateKey + LifecycleConfiguration *data.LifecycleConfiguration + Epoch uint64 +} + +type JobProvider struct { + userFetcher UserFetcher + containerFetcher ContainerFetcher + treeFetcher TreeFetcher + configurationFetcher ConfigurationFetcher + credentialSource CredentialSource + settings Settings + currentLifecycler *keys.PrivateKey + log *zap.Logger + cancelCurrentFetch context.CancelFunc + + jobChan chan Job + epochChan <-chan uint64 +} + +type Settings interface { + ServicesKeys() keys.PublicKeys +} + +type Config struct { + UserFetcher UserFetcher + ContainerFetcher ContainerFetcher + ConfigurationFetcher ConfigurationFetcher + CredentialSource CredentialSource + TreeFetcher TreeFetcher + Settings Settings + CurrentLifecycler *keys.PrivateKey + Logger *zap.Logger + BufferSize int + EpochChannel <-chan uint64 +} + +func NewJobProvider(ctx context.Context, cfg Config) *JobProvider { + provider := &JobProvider{ + userFetcher: cfg.UserFetcher, + settings: cfg.Settings, + log: cfg.Logger, + containerFetcher: cfg.ContainerFetcher, + treeFetcher: cfg.TreeFetcher, + configurationFetcher: cfg.ConfigurationFetcher, + credentialSource: cfg.CredentialSource, + currentLifecycler: cfg.CurrentLifecycler, + epochChan: cfg.EpochChannel, + jobChan: make(chan Job, cfg.BufferSize), + cancelCurrentFetch: func() {}, + } + + go provider.startFetchRoutine(ctx) + + return provider +} + +type objToHRW struct { + epoch uint64 + hash util.Uint160 +} + +func (o objToHRW) bytes() []byte { + buf := make([]byte, binary.MaxVarintLen64) + ln := binary.PutUvarint(buf, o.epoch) + + return append(o.hash[:], buf[:ln]...) +} + +type UserContainer struct { + ID user.ID + Key *keys.PrivateKey + Container cid.ID + APEChain ape.Chain +} + +func (p *JobProvider) Jobs() <-chan Job { + return p.jobChan +} + +func (p *JobProvider) startFetchRoutine(ctx context.Context) { + var ( + epochCtx context.Context + wg sync.WaitGroup + ) + + defer func() { + wg.Wait() + close(p.jobChan) + }() + + for { + select { + case <-ctx.Done(): + p.log.Info(logs.JobProviderStopped, zap.Error(ctx.Err())) + p.cancelCurrentFetch() + return + case epoch, ok := <-p.epochChan: + if !ok { + p.log.Info(logs.JobProviderStoppedBecauseOfEpochChan) + return + } + + p.log.Info(logs.FetcherTriggerEpoch, zap.Uint64("epoch", epoch)) + + p.cancelCurrentFetch() + wg.Wait() + epochCtx, p.cancelCurrentFetch = context.WithCancel(ctx) + + wg.Add(1) + go p.handleEpoch(epochCtx, epoch, &wg) + } + } +} + +func (p *JobProvider) handleEpoch(ctx context.Context, epoch uint64, wg *sync.WaitGroup) { + defer wg.Done() + + userHashes, err := p.userFetcher.Users() + if err != nil { + p.log.Error(logs.FailedToFetchUsers, zap.Error(err)) + return + } + + lifecyclers, currentPosition := p.svcKeys() + + indexes := make([]uint64, len(lifecyclers)) + for i := range indexes { + indexes[i] = uint64(i) + } + + obj := objToHRW{epoch: epoch} + for i := range userHashes { + obj.hash = userHashes[i] + h := hrw.Hash(obj.bytes()) + if hrw.Sort(indexes, h)[0] != currentPosition { + continue + } + + select { + case <-ctx.Done(): + return + default: + if err = p.handleUser(ctx, userHashes[i], epoch); err != nil { + p.log.Warn(logs.FailedToHandleUser, + zap.String("address", address.Uint160ToString(userHashes[i])), + zap.Error(err)) + } + } + } +} + +func (p *JobProvider) handleUser(ctx context.Context, userHash util.Uint160, epoch uint64) error { + userKey, err := p.resolveUserKey(ctx, userHash) + if err != nil { + return fmt.Errorf("resolve key: %w", err) + } + + var userID user.ID + user.IDFromKey(&userID, (ecdsa.PublicKey)(*userKey.PublicKey())) + + containers, err := p.containerFetcher.Containers(userID) + if err != nil { + return fmt.Errorf("list user containers: %w", err) + } + + p.log.Info(logs.FoundUserContainers, + zap.String("user", userID.EncodeToString()), + zap.Int("containers", len(containers))) + + successfullyFetchedContainers := len(containers) + + allowedChainRaw := formAllowedAPEChain(userKey.PublicKey()).Bytes() + for _, container := range containers { + uc := &UserContainer{ + ID: userID, + Key: userKey, + Container: container, + APEChain: ape.Chain{Raw: allowedChainRaw}, + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + if err = p.handleContainer(ctx, uc, epoch); err != nil { + p.log.Warn(logs.FailedToHandleContainer, + zap.String("user", userID.EncodeToString()), + zap.String("cid", container.EncodeToString()), + zap.Error(err)) + successfullyFetchedContainers-- + } + } + } + + p.log.Info(logs.FetchedUserContainers, + zap.String("user", userID.EncodeToString()), + zap.Int("successful", successfullyFetchedContainers), + zap.Int("all", len(containers))) + + return nil +} + +func (p *JobProvider) handleContainer(ctx context.Context, uc *UserContainer, epoch uint64) error { + var lifecyclerOwner user.ID + user.IDFromKey(&lifecyclerOwner, p.currentLifecycler.PrivateKey.PublicKey) // consider pre-compute this + + bktInfo := &data.BucketInfo{ + CID: uc.Container, + Owner: uc.ID, + } + + apeOverride := formAPEOverride(uc) + btoken, err := formBearerToken(epoch, apeOverride, uc.Key, lifecyclerOwner) + if err != nil { + return fmt.Errorf("form bearer token: %w", err) + } + + ctx = addBearerToContext(ctx, btoken) + objID, err := p.treeFetcher.GetBucketLifecycleConfiguration(ctx, bktInfo) + if err != nil { + return fmt.Errorf("get lifecycle configuration from tree: %w", err) + } + + var addr oid.Address + addr.SetContainer(uc.Container) + addr.SetObject(objID) + + configuration, err := p.configurationFetcher.LifecycleConfiguration(ctx, addr) + if err != nil { + return fmt.Errorf("get lifecycle configuration from storage: %w", err) + } + + job := Job{ + ContainerID: uc.Container, + PrivateKey: uc.Key, + LifecycleConfiguration: configuration, + Epoch: epoch, + } + + select { + case <-ctx.Done(): + return ctx.Err() + case p.jobChan <- job: + } + + return nil +} + +func (p *JobProvider) resolveUserKey(ctx context.Context, userHash util.Uint160) (*keys.PrivateKey, error) { + userKey, err := p.userFetcher.UserKey(userHash) + if err != nil { + return nil, fmt.Errorf("get public key: %w", err) + } + + privateKey, err := p.credentialSource.Credentials(ctx, userKey) + if err != nil { + return nil, fmt.Errorf("get private key: %w", err) + } + + return privateKey, nil +} + +func (p *JobProvider) svcKeys() (keys.PublicKeys, uint64) { + currentPublicKey := p.currentLifecycler.PublicKey() + + lifecyclerKeys := p.settings.ServicesKeys() + if position := slices.IndexFunc(lifecyclerKeys, func(pk *keys.PublicKey) bool { + return pk.Equal(currentPublicKey) + }); position == -1 { + lifecyclerKeys = append(lifecyclerKeys, currentPublicKey) + } + + sort.Slice(lifecyclerKeys, func(i, j int) bool { + return lifecyclerKeys[i].Cmp(lifecyclerKeys[j]) == -1 + }) + + position := slices.IndexFunc(lifecyclerKeys, func(pk *keys.PublicKey) bool { + return pk.Equal(currentPublicKey) + }) + if position == -1 { + // should never happen + panic("current lifecycler key isn't in list") + } + + return lifecyclerKeys, uint64(position) +} + +func formAllowedAPEChain(userKey *keys.PublicKey) *chain.Chain { + return &chain.Chain{ + ID: chain.ID("lifecycler"), + Rules: []chain.Rule{{ + Status: chain.Allow, + Actions: chain.Actions{Names: []string{"*"}}, + Resources: chain.Resources{Names: []string{"*"}}, + Condition: []chain.Condition{{ + Op: chain.CondStringEquals, + Kind: chain.KindRequest, + Key: native.PropertyKeyActorPublicKey, + Value: hex.EncodeToString(userKey.Bytes()), + }}, + }}, + } +} + +func formBearerToken(epoch uint64, apeOverride bearer.APEOverride, userKey *keys.PrivateKey, lifecyclerOwner user.ID) (*bearer.Token, error) { + var btoken bearer.Token + btoken.SetIat(epoch) + btoken.SetNbf(epoch) + btoken.SetExp(epoch + 2) // maybe +1, I'm not sure if we should configure this parameter + btoken.SetAPEOverride(apeOverride) + btoken.AssertUser(lifecyclerOwner) + + if err := btoken.Sign(userKey.PrivateKey); err != nil { + return nil, fmt.Errorf("sign: %w", err) + } + + return &btoken, nil +} + +func formAPEOverride(userInfo *UserContainer) bearer.APEOverride { + return bearer.APEOverride{ + Target: ape.ChainTarget{ + TargetType: ape.TargetTypeContainer, + Name: userInfo.Container.EncodeToString(), + }, + Chains: []ape.Chain{userInfo.APEChain}, + } +} + +func addBearerToContext(ctx context.Context, btoken *bearer.Token) context.Context { + return middleware.SetBox(ctx, &middleware.Box{ + AccessBox: &accessbox.Box{ + Gate: &accessbox.GateData{ + BearerToken: btoken, + }, + }, + }) +} diff --git a/internal/lifecycle/fetcher_test.go b/internal/lifecycle/fetcher_test.go new file mode 100644 index 0000000..1715d94 --- /dev/null +++ b/internal/lifecycle/fetcher_test.go @@ -0,0 +1,305 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +var _ UserFetcher = (*userFetcherMock)(nil) + +type userFetcherMock struct { + users map[util.Uint160]*keys.PrivateKey +} + +func newUserFetcherMock(users map[util.Uint160]*keys.PrivateKey) *userFetcherMock { + if users == nil { + users = map[util.Uint160]*keys.PrivateKey{} + } + return &userFetcherMock{ + users: users, + } +} + +func (u *userFetcherMock) Users() ([]util.Uint160, error) { + res := make([]util.Uint160, 0, len(u.users)) + + for hash := range u.users { + res = append(res, hash) + } + + return res, nil +} + +func (u *userFetcherMock) UserKey(hash util.Uint160) (*keys.PublicKey, error) { + key, ok := u.users[hash] + if !ok { + return nil, errors.New("userFetcherMock: hash not found") + } + + return key.PublicKey(), nil +} + +var _ ContainerFetcher = (*containerFetcherMock)(nil) + +type containerFetcherMock struct { + containers map[util.Uint160][]cid.ID +} + +func newContainerFetcherMock(containers map[util.Uint160][]cid.ID) *containerFetcherMock { + if containers == nil { + containers = map[util.Uint160][]cid.ID{} + } + return &containerFetcherMock{ + containers: containers, + } +} + +func (c *containerFetcherMock) Containers(owner user.ID) ([]cid.ID, error) { + hash, err := owner.ScriptHash() + if err != nil { + return nil, err + } + + containers, ok := c.containers[hash] + if !ok { + return nil, errors.New("containerFetcherMock: hash not found") + } + + return containers, nil +} + +var _ ConfigurationFetcher = (*configurationFetcherMock)(nil) + +type configurationFetcherMock struct { + configurations map[oid.Address]*data.LifecycleConfiguration +} + +func newConfigurationFetcherMock(configs map[oid.Address]*data.LifecycleConfiguration) *configurationFetcherMock { + if configs == nil { + configs = map[oid.Address]*data.LifecycleConfiguration{} + } + return &configurationFetcherMock{ + configurations: configs, + } +} + +func (c *configurationFetcherMock) LifecycleConfiguration(_ context.Context, addr oid.Address) (*data.LifecycleConfiguration, error) { + val, ok := c.configurations[addr] + if !ok { + return nil, errors.New("configurationFetcherMock: hash not found") + } + + return val, nil +} + +var _ CredentialSource = (*credentialSourceMock)(nil) + +type credentialSourceMock struct { + users map[util.Uint160]*keys.PrivateKey +} + +func newCredentialSourceMock(users map[util.Uint160]*keys.PrivateKey) *credentialSourceMock { + if users == nil { + users = map[util.Uint160]*keys.PrivateKey{} + } + return &credentialSourceMock{ + users: users, + } +} + +func (c *credentialSourceMock) Credentials(_ context.Context, pk *keys.PublicKey) (*keys.PrivateKey, error) { + key, ok := c.users[pk.GetScriptHash()] + if !ok { + return nil, errors.New("credentialSourceMock: hash not found") + } + + return key, nil +} + +var _ TreeFetcher = (*treeFetcherMock)(nil) + +type treeFetcherMock struct { + configurations map[cid.ID]oid.ID +} + +func newTreeFetcherMock(configs map[cid.ID]oid.ID) *treeFetcherMock { + if configs == nil { + configs = map[cid.ID]oid.ID{} + } + return &treeFetcherMock{ + configurations: configs, + } +} + +func (t *treeFetcherMock) GetBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) (oid.ID, error) { + val, ok := t.configurations[bktInfo.CID] + if !ok { + return oid.ID{}, errors.New("treeFetcherMock: hash not found") + } + + return val, nil +} + +var _ Settings = (*settingsMock)(nil) + +type settingsMock struct{} + +func (s *settingsMock) ServicesKeys() keys.PublicKeys { + return nil +} + +func TestFetcherBase(t *testing.T) { + ctx := context.Background() + log := zaptest.NewLogger(t) + + key, err := keys.NewPrivateKey() + require.NoError(t, err) + + mocks, err := initMocks(2, 1) + require.NoError(t, err) + + epochCh := make(chan uint64) + go func() { + epochCh <- 1 + close(epochCh) + }() + + cfg := Config{ + UserFetcher: mocks.userFetcher, + ContainerFetcher: mocks.containerFetcher, + ConfigurationFetcher: mocks.configurationFetcher, + CredentialSource: mocks.credentialSource, + TreeFetcher: mocks.treeFetcher, + Settings: &settingsMock{}, + CurrentLifecycler: key, + Logger: log, + EpochChannel: epochCh, + } + + f := NewJobProvider(ctx, cfg) + + var res []Job + for job := range f.Jobs() { + res = append(res, job) + } + + require.Len(t, res, 2) +} + +func TestFetcherCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + log := zaptest.NewLogger(t) + + key, err := keys.NewPrivateKey() + require.NoError(t, err) + + mocks, err := initMocks(1, 3) + require.NoError(t, err) + + epochCh := make(chan uint64) + go func() { + epochCh <- 1 + close(epochCh) + }() + + cfg := Config{ + UserFetcher: mocks.userFetcher, + ContainerFetcher: mocks.containerFetcher, + ConfigurationFetcher: mocks.configurationFetcher, + CredentialSource: mocks.credentialSource, + TreeFetcher: mocks.treeFetcher, + Settings: &settingsMock{}, + CurrentLifecycler: key, + Logger: log, + EpochChannel: epochCh, + } + + f := NewJobProvider(ctx, cfg) + + ch := f.Jobs() + + res := []Job{<-ch} + cancel() + <-ctx.Done() + + for job := range ch { + res = append(res, job) + } + + require.Len(t, res, 1) +} + +type fetchersMock struct { + userFetcher *userFetcherMock + containerFetcher *containerFetcherMock + configurationFetcher *configurationFetcherMock + credentialSource *credentialSourceMock + treeFetcher *treeFetcherMock +} + +func initMocks(users, containers int) (*fetchersMock, error) { + usersMap, err := generateUsersMap(users) + if err != nil { + return nil, err + } + + cnrsMap := make(map[util.Uint160][]cid.ID) + treeMap := make(map[cid.ID]oid.ID) + configMap := make(map[oid.Address]*data.LifecycleConfiguration) + for hash := range usersMap { + for i := 0; i < containers; i++ { + addr := oidtest.Address() + cnrsMap[hash] = append(cnrsMap[hash], addr.Container()) + treeMap[addr.Container()] = addr.Object() + configMap[addr] = &data.LifecycleConfiguration{Rules: []data.LifecycleRule{{ID: addr.EncodeToString()}}} + } + } + + return &fetchersMock{ + userFetcher: newUserFetcherMock(usersMap), + containerFetcher: newContainerFetcherMock(cnrsMap), + configurationFetcher: newConfigurationFetcherMock(configMap), + credentialSource: newCredentialSourceMock(usersMap), + treeFetcher: newTreeFetcherMock(treeMap), + }, nil +} + +func generateKeys(n int) ([]*keys.PrivateKey, error) { + var err error + res := make([]*keys.PrivateKey, n) + + for i := 0; i < n; i++ { + if res[i], err = keys.NewPrivateKey(); err != nil { + return nil, err + } + } + + return res, nil +} + +func generateUsersMap(n int) (map[util.Uint160]*keys.PrivateKey, error) { + res := make(map[util.Uint160]*keys.PrivateKey, n) + + userKeys, err := generateKeys(n) + if err != nil { + return nil, err + } + + for _, key := range userKeys { + res[key.GetScriptHash()] = key + } + + return res, nil +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 076c483..b6e63ea 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -15,11 +15,36 @@ const ( FailedToReloadConfig = "failed to reload config" LogLevelWontBeUpdated = "log level won't be updated" SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" - NotificatorStopped = "notificator stopped" + ListenerStopped = "listener stopped" + MorphClientStopped = "morph client stopped" + MorphClientReconnection = "morph client reconnection..." + ListenerReconnection = "listener reconnection..." + MorphClientCouldntBeReconnected = "morph client couldn't be reconnected" + ListenerCouldntBeReconnected = "listener couldn't be reconnected" ResolveNetmapContract = "failed to resolve netmap contract" + ResolveFrostfsIDContract = "failed to resolve frostfsid contract" + ResolveContainerContract = "failed to resolve container contract" NewEpochWasTriggered = "new epoch was triggered" - ListenerCouldntBeReinitialized = "listener couldn't be reinitialized" InitNotificator = "init notificator" NoMorphRPCEndpoints = "no morph RPC endpoints" FailedToLoadPrivateKey = "failed to load private key" + NoCredentialSourceWallets = "no credential source wallets" + CouldntCreateWalletSource = "could not create wallet source" + AddedStoragePeer = "added storage peer" + FailedToCreateConnectionPool = "failed to create connection pool" + FailedToDialConnectionPool = "failed to dial connection pool" + FailedToCreateTreePool = "failed to create tree pool" + FailedToDialTreePool = "failed to dial tree pool" + FoundUserContainers = "found user containers" + JobProviderStopped = "job provider stopped" + JobProviderStoppedBecauseOfEpochChan = "job provider stopped because of epoch channel is closed" + FailedToInitMorphClient = "failed to init morph client" + FailedToFetchServicesKeys = "failed to fetch lifecycle services keys" + FailedToFetchUsers = "failed to fetch users" + FailedToHandleUser = "failed to handle user" + FailedToHandleContainer = "failed to handle container" + FetcherTriggerEpoch = "fetcher: trigger epoch, cancel previous fetch" + FetchedUserContainers = "fetched user container configurations" + HandlerTriggered = "handler: triggered" + HandlerContextCanceled = "handler: context canceled" ) diff --git a/internal/morph/client.go b/internal/morph/client.go new file mode 100644 index 0000000..43fa987 --- /dev/null +++ b/internal/morph/client.go @@ -0,0 +1,119 @@ +package morph + +import ( + "context" + "fmt" + "sync" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "go.uber.org/zap" +) + +type Client struct { + mu sync.RWMutex + client *client.Client + + clientOptions []client.Option + log *zap.Logger + key *keys.PrivateKey + connLost chan struct{} + reconnectInterval time.Duration + reconnection chan struct{} +} + +type Config struct { + Logger *zap.Logger + Endpoints []client.Endpoint + Key *keys.PrivateKey + ReconnectInterval time.Duration + DialTimeout time.Duration +} + +func New(ctx context.Context, cfg Config) (*Client, error) { + c := &Client{ + log: cfg.Logger, + key: cfg.Key, + connLost: make(chan struct{}), + reconnectInterval: cfg.ReconnectInterval, + reconnection: make(chan struct{}), + } + + c.clientOptions = []client.Option{ + client.WithLogger(&logger.Logger{Logger: cfg.Logger}), + client.WithEndpoints(cfg.Endpoints...), + client.WithConnLostCallback(func() { c.connLost <- struct{}{} }), + client.WithDialTimeout(cfg.DialTimeout), + } + + if err := c.initNewClient(ctx); err != nil { + return nil, err + } + + go c.reconnectRoutine(ctx) + + return c, nil +} + +func (c *Client) reconnectRoutine(ctx context.Context) { + ticker := time.NewTicker(c.reconnectInterval) + defer func() { + ticker.Stop() + close(c.connLost) + close(c.reconnection) + }() + + for { + select { + case <-ctx.Done(): + c.log.Info(logs.MorphClientStopped, zap.Error(ctx.Err())) + return + case <-c.connLost: + c.Client().Close() + LOOP: + for { + select { + case <-ctx.Done(): + c.log.Info(logs.MorphClientStopped, zap.Error(ctx.Err())) + return + case <-ticker.C: + c.log.Info(logs.MorphClientReconnection) + if err := c.initNewClient(ctx); err != nil { + c.log.Error(logs.MorphClientCouldntBeReconnected, zap.Error(err)) + ticker.Reset(c.reconnectInterval) + continue + } + c.reconnection <- struct{}{} + break LOOP + } + } + } + } +} + +func (c *Client) initNewClient(ctx context.Context) error { + cli, err := client.New(ctx, c.key, c.clientOptions...) + if err != nil { + return fmt.Errorf("create new client: %w", err) + } + + c.mu.Lock() + c.client = cli + c.mu.Unlock() + + return nil +} + +func (c *Client) Client() *client.Client { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.client +} + +func (c *Client) ReconnectionChannel() <-chan struct{} { + return c.reconnection +} diff --git a/internal/morph/contract/container.go b/internal/morph/contract/container.go new file mode 100644 index 0000000..e1a24a1 --- /dev/null +++ b/internal/morph/contract/container.go @@ -0,0 +1,70 @@ +package contract + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-contract/commonclient" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/morph" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "go.uber.org/zap" +) + +type Container struct { + client *morph.Client + contractHash util.Uint160 + log *zap.Logger +} + +type ContainerConfig struct { + Client *morph.Client + ContractHash util.Uint160 + Log *zap.Logger +} + +const ( + batchSize = 100 + + containersOfMethod = "containersOf" +) + +func NewContainer(cfg ContainerConfig) *Container { + return &Container{ + client: cfg.Client, + contractHash: cfg.ContractHash, + log: cfg.Log, + } +} + +func (c *Container) Containers(ownerID user.ID) ([]cid.ID, error) { + items, err := commonclient.ReadIteratorItems(c.client.Client().GetActor(), batchSize, c.contractHash, containersOfMethod, ownerID.WalletBytes()) + if err != nil { + return nil, fmt.Errorf("read iterator items (%s): %w", containersOfMethod, err) + } + + cidList, err := decodeCID(items) + if err != nil { + return nil, err + } + + return cidList, nil +} + +func decodeCID(items []stackitem.Item) ([]cid.ID, error) { + cidList := make([]cid.ID, len(items)) + for i, item := range items { + rawID, err := client.BytesFromStackItem(item) + if err != nil { + return nil, fmt.Errorf("could not get byte array from stack item: %w", err) + } + + if err = cidList[i].Decode(rawID); err != nil { + return nil, fmt.Errorf("decode container id: %w", err) + } + } + + return cidList, nil +} diff --git a/internal/morph/contract/frostfsid.go b/internal/morph/contract/frostfsid.go new file mode 100644 index 0000000..8bf8c23 --- /dev/null +++ b/internal/morph/contract/frostfsid.go @@ -0,0 +1,92 @@ +package contract + +import ( + "errors" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + morphclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/morph" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +type FrostFSID struct { + morphClient *morph.Client + contractHash util.Uint160 + + mu sync.RWMutex + cli *client.Client +} + +type FrostFSIDConfig struct { + // Client is a multi neo-go client with auto reconnect. + Client *morph.Client + + // Contract is hash of contract. + ContractHash util.Uint160 +} + +// NewFrostFSID creates new FrostfsID contract wrapper. +func NewFrostFSID(cfg FrostFSIDConfig) *FrostFSID { + ffsid := &FrostFSID{ + morphClient: cfg.Client, + contractHash: cfg.ContractHash, + cli: client.NewSimple(cfg.Client.Client().GetActor(), cfg.ContractHash), + } + + return ffsid +} + +func (f *FrostFSID) Users() ([]util.Uint160, error) { + var res []util.Uint160 + err := f.requestWithRetryOnConnectionLost(func(c *client.Client) error { + var inErr error + res, inErr = c.ListSubjects() + return inErr + }) + + return res, err +} + +func (f *FrostFSID) UserKey(hash util.Uint160) (*keys.PublicKey, error) { + var res *client.Subject + err := f.requestWithRetryOnConnectionLost(func(c *client.Client) error { + var inErr error + res, inErr = c.GetSubject(hash) + return inErr + }) + if err != nil { + return nil, err + } + + return res.PrimaryKey, nil +} + +func (f *FrostFSID) requestWithRetryOnConnectionLost(fn func(c *client.Client) error) error { + err := fn(f.client()) + if err == nil { + return nil + } + + if !errors.Is(err, morphclient.ErrConnectionLost) { + return err + } + + f.initNewClient() + + return fn(f.client()) +} + +func (f *FrostFSID) initNewClient() { + f.mu.Lock() + f.cli = client.NewSimple(f.morphClient.Client().GetActor(), f.contractHash) + f.mu.Unlock() +} + +func (f *FrostFSID) client() *client.Client { + f.mu.RLock() + defer f.mu.RUnlock() + + return f.cli +} diff --git a/internal/notificator/limiter.go b/internal/notificator/limiter.go index e755f07..94839c9 100644 --- a/internal/notificator/limiter.go +++ b/internal/notificator/limiter.go @@ -64,7 +64,7 @@ func (h *handlerLimiter) Handler(e event.Event) { } workCtx := h.replaceCurrentWorkContext(h.ctx) - h.log.Debug(logs.NewEpochWasTriggered, zap.Int64("epoch", ee.Epoch)) + h.log.Debug(logs.NewEpochWasTriggered, zap.Uint64("epoch", ee.Epoch)) h.work <- func() { h.handler(workCtx, ee) } diff --git a/internal/notificator/listener.go b/internal/notificator/listener.go new file mode 100644 index 0000000..bd05b29 --- /dev/null +++ b/internal/notificator/listener.go @@ -0,0 +1,154 @@ +package notificator + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/morph" + "github.com/nspcc-dev/neo-go/pkg/core/block" + "go.uber.org/zap" +) + +type ListenerImpl struct { + client *morph.Client + log *zap.Logger + reconnectInterval time.Duration + parser event.NotificationParserInfo + handler event.NotificationHandlerInfo + + blockNumber atomic.Uint32 + + once sync.Once + mu sync.RWMutex + listener event.Listener +} + +type ConfigListener struct { + Client *morph.Client + Logger *zap.Logger + ReconnectInterval time.Duration + Parser event.NotificationParserInfo + Handler event.NotificationHandlerInfo +} + +var _ Listener = (*ListenerImpl)(nil) + +func NewListener(ctx context.Context, cfg ConfigListener) (*ListenerImpl, error) { + l := &ListenerImpl{ + client: cfg.Client, + log: cfg.Logger, + reconnectInterval: cfg.ReconnectInterval, + parser: cfg.Parser, + handler: cfg.Handler, + } + + if err := l.initNewListener(ctx); err != nil { + return nil, err + } + + return l, nil +} + +func (l *ListenerImpl) Listen(ctx context.Context) { + l.once.Do(func() { + l.setParsersAndHandlers() + go l.currentListener().Listen(ctx) + l.reconnectRoutine(ctx) + }) +} + +func (l *ListenerImpl) reconnectRoutine(ctx context.Context) { + ticker := time.NewTicker(l.reconnectInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + l.log.Info(logs.ListenerStopped, zap.Error(ctx.Err())) + return + case <-l.client.ReconnectionChannel(): + LOOP: + for { + select { + case <-ctx.Done(): + l.log.Info(logs.ListenerStopped, zap.Error(ctx.Err())) + return + case <-ticker.C: + l.log.Info(logs.ListenerReconnection) + if err := l.initNewListener(ctx); err != nil { + l.log.Error(logs.ListenerCouldntBeReconnected, zap.Error(err)) + ticker.Reset(l.reconnectInterval) + continue + } + l.setParsersAndHandlers() + go l.currentListener().Listen(ctx) + break LOOP + } + } + } + } +} + +func (l *ListenerImpl) initNewListener(ctx context.Context) error { + currentBlock, err := l.client.Client().BlockCount() + if err != nil { + return fmt.Errorf("get block count: %w", err) + } + + latestBlock := l.blockNumber.Load() + if currentBlock > latestBlock { + latestBlock = currentBlock + } + + morphLogger := &logger.Logger{Logger: l.log} + + subs, err := subscriber.New(ctx, &subscriber.Params{ + Log: morphLogger, + StartFromBlock: latestBlock, + Client: l.client.Client(), + }) + if err != nil { + return fmt.Errorf("create subscriber: %w", err) + } + + ln, err := event.NewListener(event.ListenerParams{ + Logger: morphLogger, + Subscriber: subs, + WorkerPoolCapacity: 0, // 0 means "infinite" + }) + if err != nil { + return err + } + + l.mu.Lock() + l.listener = ln + l.mu.Unlock() + + return nil +} + +func (l *ListenerImpl) currentListener() event.Listener { + l.mu.RLock() + defer l.mu.RUnlock() + return l.listener +} + +func (l *ListenerImpl) setParsersAndHandlers() { + l.mu.RLock() + defer l.mu.RUnlock() + + l.listener.SetNotificationParser(l.parser) + l.listener.RegisterNotificationHandler(l.handler) + l.listener.RegisterBlockHandler(l.blockHandler) +} + +func (l *ListenerImpl) blockHandler(block *block.Block) { + l.blockNumber.Store(block.Index) +} diff --git a/internal/notificator/notificator.go b/internal/notificator/notificator.go index 6f1c347..ae262b9 100644 --- a/internal/notificator/notificator.go +++ b/internal/notificator/notificator.go @@ -3,10 +3,8 @@ package notificator import ( "context" "fmt" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "git.frostfs.info/TrueCloudLab/frostfs-s3-lifecycler/internal/logs" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" @@ -16,107 +14,72 @@ import ( type NewEpochHandler func(ctx context.Context, ee NewEpochEvent) type NewEpochEvent struct { - Epoch int64 + Epoch uint64 } func (n NewEpochEvent) MorphEvent() {} -type ListenerCreationFunc func(connectionLostCallback func()) (event.Listener, error) +type Listener interface { + // Listen must start the event listener. + // + // Must listen to events with the parser installed. + Listen(context.Context) +} + +type ListenerConfig struct { + Parser event.NotificationParserInfo + Handler event.NotificationHandlerInfo +} type Notificator struct { - logger *zap.Logger - listener event.Listener - handler *handlerLimiter - connLost chan struct{} - netmapContract util.Uint160 - newListener ListenerCreationFunc - reconnectClientsInterval time.Duration + logger *zap.Logger + listener Listener + handler *handlerLimiter } type Config struct { - Handler NewEpochHandler - Logger *zap.Logger - NewListener ListenerCreationFunc - NetmapContract util.Uint160 - ReconnectClientsInterval time.Duration + Handler NewEpochHandler + Logger *zap.Logger + NewListenerFn func(ListenerConfig) (Listener, error) + NetmapContract util.Uint160 } const newEpochEventType = event.Type("NewEpoch") func New(ctx context.Context, cfg Config) (*Notificator, error) { notifier := &Notificator{ - netmapContract: cfg.NetmapContract, - handler: newHandlerLimiter(ctx, cfg.Handler, cfg.Logger), - connLost: make(chan struct{}), - newListener: cfg.NewListener, - logger: cfg.Logger, - reconnectClientsInterval: cfg.ReconnectClientsInterval, - } - - if err := notifier.initListener(); err != nil { - return nil, fmt.Errorf("init listener: %w", err) - } - - return notifier, nil -} - -func (n *Notificator) initListener() error { - listener, err := n.newListener(func() { n.connLost <- struct{}{} }) - if err != nil { - return err + handler: newHandlerLimiter(ctx, cfg.Handler, cfg.Logger), + logger: cfg.Logger, } var npi event.NotificationParserInfo - npi.SetScriptHash(n.netmapContract) + npi.SetScriptHash(cfg.NetmapContract) npi.SetType(newEpochEventType) npi.SetParser(newEpochEventParser()) - listener.SetNotificationParser(npi) var nhi event.NotificationHandlerInfo nhi.SetType(newEpochEventType) - nhi.SetScriptHash(n.netmapContract) - nhi.SetHandler(n.handler.Handler) - listener.RegisterNotificationHandler(nhi) + nhi.SetScriptHash(cfg.NetmapContract) + nhi.SetHandler(notifier.handler.Handler) - n.listener = listener + ln, err := cfg.NewListenerFn(ListenerConfig{ + Parser: npi, + Handler: nhi, + }) + if err != nil { + return nil, fmt.Errorf("create new listener: %w", err) + } - return nil + notifier.listener = ln + + return notifier, nil } // Start runs listener to process notifications. // Method MUST be invoked once after successful initialization with New // otherwise panic can happen. func (n *Notificator) Start(ctx context.Context) { - go n.listener.Listen(ctx) - - ticker := time.NewTicker(n.reconnectClientsInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - n.logger.Info(logs.NotificatorStopped, zap.Error(ctx.Err())) - return - case <-n.connLost: - n.listener.Stop() - LOOP: - for { - select { - case <-ctx.Done(): - n.logger.Info(logs.NotificatorStopped, zap.Error(ctx.Err())) - return - case <-ticker.C: - if err := n.initListener(); err != nil { - n.logger.Error(logs.ListenerCouldntBeReinitialized, zap.Error(err)) - ticker.Reset(n.reconnectClientsInterval) - continue - } - go n.listener.Listen(ctx) - break LOOP - } - } - } - } + n.listener.Listen(ctx) } func newEpochEventParser() event.NotificationParser { @@ -134,7 +97,7 @@ func newEpochEventParser() event.NotificationParser { return nil, err } - return NewEpochEvent{Epoch: epoch.Int64()}, nil + return NewEpochEvent{Epoch: epoch.Uint64()}, nil } } diff --git a/internal/notificator/notificator_test.go b/internal/notificator/notificator_test.go index a88c95c..a6cf399 100644 --- a/internal/notificator/notificator_test.go +++ b/internal/notificator/notificator_test.go @@ -2,13 +2,11 @@ package notificator import ( "context" - "errors" "sync" "sync/atomic" "testing" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/require" @@ -24,12 +22,8 @@ type scriptHashWithType struct { type listenerMock struct { scriptHashWithType - mu sync.Mutex - parsers map[scriptHashWithType]event.NotificationParserInfo - handlers map[scriptHashWithType][]event.NotificationHandlerInfo - started, stopped bool - - lostConnectionCallback func() + parser event.NotificationParserInfo + handler event.NotificationHandlerInfo } func newListenerMock(hash util.Uint160) *listenerMock { @@ -38,97 +32,15 @@ func newListenerMock(hash util.Uint160) *listenerMock { eventType: newEpochEventType, contractHash: hash, }, - parsers: map[scriptHashWithType]event.NotificationParserInfo{}, - handlers: map[scriptHashWithType][]event.NotificationHandlerInfo{}, - started: false, - stopped: false, } } func (l *listenerMock) sendNotification(epochEvent NewEpochEvent) error { - l.mu.Lock() - defer l.mu.Unlock() - - if _, ok := l.parsers[l.scriptHashWithType]; !ok { - return errors.New("there is no appropriate parser") - } - - handlers, ok := l.handlers[l.scriptHashWithType] - if !ok { - return errors.New("there is no appropriate handlers") - } - - for _, handler := range handlers { - handler.Handler()(epochEvent) - } - + l.handler.Handler()(epochEvent) return nil } -func (l *listenerMock) refresh() { - l.mu.Lock() - defer l.mu.Unlock() - - l.started = false - l.stopped = false - l.parsers = map[scriptHashWithType]event.NotificationParserInfo{} - l.handlers = map[scriptHashWithType][]event.NotificationHandlerInfo{} -} - -func (l *listenerMock) Listen(context.Context) { - l.mu.Lock() - l.started = true - l.mu.Unlock() -} - -func (l *listenerMock) ListenWithError(context.Context, chan<- error) { - panic("not implemented") -} - -func (l *listenerMock) SetNotificationParser(info event.NotificationParserInfo) { - l.mu.Lock() - defer l.mu.Unlock() - - l.parsers[scriptHashWithType{ - eventType: info.GetType(), - contractHash: info.ScriptHash(), - }] = info -} - -func (l *listenerMock) RegisterNotificationHandler(info event.NotificationHandlerInfo) { - l.mu.Lock() - defer l.mu.Unlock() - - key := scriptHashWithType{ - eventType: info.GetType(), - contractHash: info.ScriptHash(), - } - list := l.handlers[key] - - l.handlers[key] = append(list, info) -} - -func (l *listenerMock) EnableNotarySupport(util.Uint160, client.AlphabetKeys, event.BlockCounter) { - panic("not implemented") -} - -func (l *listenerMock) SetNotaryParser(event.NotaryParserInfo) { - panic("not implemented") -} - -func (l *listenerMock) RegisterNotaryHandler(event.NotaryHandlerInfo) { - panic("not implemented") -} - -func (l *listenerMock) RegisterBlockHandler(event.BlockHandler) { - panic("not implemented") -} - -func (l *listenerMock) Stop() { - l.mu.Lock() - l.stopped = true - l.mu.Unlock() -} +func (l *listenerMock) Listen(context.Context) {} func TestNotificatorBase(t *testing.T) { ctx := context.Background() @@ -152,15 +64,14 @@ func TestNotificatorBase(t *testing.T) { lnMock := newListenerMock(contractHash) cfg := Config{ - Handler: handler, - Logger: logger, - NewListener: func(cb func()) (event.Listener, error) { - lnMock.lostConnectionCallback = cb - lnMock.refresh() + Handler: handler, + Logger: logger, + NetmapContract: contractHash, + NewListenerFn: func(config ListenerConfig) (Listener, error) { + lnMock.parser = config.Parser + lnMock.handler = config.Handler return lnMock, nil }, - NetmapContract: contractHash, - ReconnectClientsInterval: 100 * time.Millisecond, } n, err := New(ctx, cfg) @@ -175,12 +86,6 @@ func TestNotificatorBase(t *testing.T) { ee = NewEpochEvent{Epoch: 2} sendNotification(t, lnMock, ee, &wg) require.Equal(t, ee.Epoch, gotEvent.Epoch) - - lnMock.lostConnectionCallback() - - ee = NewEpochEvent{Epoch: 3} - sendNotification(t, lnMock, ee, &wg) - require.Equal(t, ee.Epoch, gotEvent.Epoch) } func sendNotification(t *testing.T, lnMock *listenerMock, ee NewEpochEvent, wg *sync.WaitGroup) {